{ "runId": "2026-07-23T10-51-51-233Z-cost-per-task-openrouter", "generatedAt": "2026-07-23T10:59:14.092Z", "experiment": "cost-per-task", "adapter": "openrouter", "methodology": { "inferenceRoute": "OpenRouter chat completions", "costBasis": "AI Pricing Guru direct-provider list prices from /opt/aipricing/repo/public/pricing.json", "grading": "deterministic exact/number/regex/contains/json_fields graders; no LLM judge", "monthlyCapUsd": 100, "runBudgetUsd": 5, "maxOutputTokens": 1024, "concurrency": 4 }, "suite": { "id": "cost-per-task-v2", "version": 2 }, "models": [ { "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna" }, { "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra" }, { "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol" }, { "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5" }, { "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8" }, { "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5" }, { "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5" }, { "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash" }, { "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite" }, { "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro" }, { "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash" }, { "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2" }, { "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3" }, { "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5" } ], "tasks": [ { "id": "math-001", "category": "reasoning", "prompt": "Return only the integer answer. A batch job processes 18 files per minute for 7 minutes, then 12 files per minute for 5 minutes. How many files total?" }, { "id": "math-002", "category": "reasoning", "prompt": "Return only the integer answer. A model costs $0.30 per 1M input tokens and $2.50 per 1M output tokens. What is the cost in cents for 100,000 input tokens and 20,000 output tokens?" }, { "id": "math-003", "category": "reasoning", "prompt": "Return only the integer answer. If a 64,000-token context window is filled to 75%, how many tokens are used?" }, { "id": "math-004", "category": "reasoning", "prompt": "Return only the integer answer. A provider raises output price from $4/M to $6/M. What is the percent increase?" }, { "id": "math-005", "category": "reasoning", "prompt": "Return only the integer answer. A cache hit costs 10% of normal input price. Normal input is $2/M. What is the cache-hit price in cents per 1M tokens?" }, { "id": "math-006", "category": "reasoning", "prompt": "Return only the integer answer. A crawler finds 9 new models Monday, 14 Tuesday, and removes 5 duplicates. How many unique new models remain?" }, { "id": "math-007", "category": "reasoning", "prompt": "Return only the integer answer. A benchmark has 50 tasks. A model gets 37 correct. What is the accuracy percentage rounded to the nearest whole number?" }, { "id": "math-008", "category": "reasoning", "prompt": "Return only the decimal answer with no currency symbol. If a run spends $0.42 and gets 14 correct answers, what is dollars per correct answer?" }, { "id": "extract-001", "category": "extraction", "prompt": "Return only JSON with numeric values for prices and context, without currency symbols or units. Text: 'Gemini Flash: input $0.30/M, output $2.50/M, context 1,048,576 tokens.' Extract fields model, inputPerM, outputPerM, context." }, { "id": "extract-002", "category": "extraction", "prompt": "Return only JSON. Text: 'Z.ai GLM-5.2 charges $1.40 per million input tokens and $4.40 per million output tokens.' Extract provider, model, inputPerM, outputPerM." }, { "id": "extract-003", "category": "extraction", "prompt": "Return only the single word immediately before 'plan'. Sentence: 'For heavy coding, the Pro plan is cheaper than per-token API use after roughly 35 million output tokens.'" }, { "id": "extract-004", "category": "extraction", "prompt": "Return only JSON. Log line: '2026-07-20 model=deepseek-v4-flash prompt=120 completion=38 latency_ms=910 ok=true'. Extract model, prompt, completion, latency_ms, ok." }, { "id": "extract-005", "category": "extraction", "prompt": "Return only a comma-separated list of model IDs in their original order. Text: 'Tested: gpt-5.6-luna; claude-sonnet-5; gemini-2.5-flash.'" }, { "id": "extract-006", "category": "extraction", "prompt": "Return only JSON. Text: 'Anthropic Claude Haiku 4.5: $1 input, $5 output, active.' Extract provider, model, status." }, { "id": "classify-001", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'OpenAI cuts GPT-5.6 Luna output token price by 20%'." }, { "id": "classify-002", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'DeepSeek introduces V4 Flash for low-latency chat apps'." }, { "id": "classify-003", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Claude API errors spike across us-east for 42 minutes'." }, { "id": "classify-004", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Step-by-step guide to add streaming responses to a chatbot'." }, { "id": "classify-005", "category": "classification", "prompt": "Return only yes or no. Query: 'cheapest ai api per token'. Is this query primarily cost/comparison intent?" }, { "id": "classify-006", "category": "classification", "prompt": "Return only yes or no. Query: 'what is a transformer neural network'. Is this query primarily pricing intent?" }, { "id": "summ-001", "category": "summarization", "prompt": "Summarize in exactly 8 words and include these exact words: cheaper, extraction, failed, arithmetic. Text: 'The cheaper model answered all extraction tasks correctly but failed most multi-step arithmetic, making it a good fit for structured parsing rather than reasoning.'" }, { "id": "summ-002", "category": "summarization", "prompt": "Return these three search phrases exactly as written, lowercase and separated by commas with no extra text: ai api pricing, cheapest ai api, token calculator. Text: 'Rising searches include ai api pricing, cheapest ai api, and token calculator, all pointing toward cost-conscious developer intent.'" }, { "id": "summ-003", "category": "summarization", "prompt": "Return one sentence under 12 words and include the exact words verbosity and cost. Text: 'A model with high accuracy but very high output verbosity can lose on dollars per correct answer because every extra token compounds cost.'" }, { "id": "summ-004", "category": "summarization", "prompt": "Return only a title under 7 words that includes the exact words cost and correct. Text: 'We compare leading AI models by how much each correct benchmark answer costs, not just accuracy.'" }, { "id": "summ-005", "category": "summarization", "prompt": "Return exactly one sentence that includes OpenRouter and the exact phrase list prices. Text: 'OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.'" }, { "id": "code-001", "category": "coding", "prompt": "Return only the output. JavaScript: const xs=[3,1,4,1,5]; console.log(xs.filter(x=>x>2).reduce((a,b)=>a+b,0));" }, { "id": "code-002", "category": "coding", "prompt": "Return only the output. Python: prices={'in':0.3,'out':2.5}; print(round(prices['in']*2 + prices['out']*0.1, 2))" }, { "id": "code-003", "category": "coding", "prompt": "Return only the missing JavaScript expression. Complete: const cost = (inputTokens * ___ + outputTokens * outputPerM) / 1_000_000;" }, { "id": "code-004", "category": "coding", "prompt": "Return only JSON. For rows [{ok:true,cost:0.02},{ok:false,cost:0.03},{ok:true,cost:0.01}], compute correct, totalCost, costPerCorrect." }, { "id": "code-005", "category": "coding", "prompt": "Return only the output. JavaScript: console.log(['gpt','claude','glm'].map(s=>s.length).join('-'));" }, { "id": "code-006", "category": "coding", "prompt": "Return only the called function name immediately after await. Snippet: async function call(){ const r = await fetch(url, opts); return r.json(); }" }, { "id": "logic-001", "category": "instruction", "prompt": "Return only the third item alphabetically: Luna, Flash, Sonnet, Grok." }, { "id": "logic-002", "category": "instruction", "prompt": "Return only the word that appears twice: token, price, model, token, output." }, { "id": "logic-003", "category": "instruction", "prompt": "Return only the reversed string: glm-5.2" }, { "id": "logic-004", "category": "instruction", "prompt": "Return only the uppercase acronym from: cost per correct answer." }, { "id": "logic-005", "category": "instruction", "prompt": "Return only a comma-separated list of the two providers with names starting with G: OpenAI, Google, Anthropic, Groq, Mistral." }, { "id": "logic-006", "category": "instruction", "prompt": "Return only valid or invalid. JSON string: {\"model\":\"gpt\",\"cost\":0.01}" }, { "id": "logic-007", "category": "instruction", "prompt": "Return only valid or invalid. JSON string: {model:\"gpt\",cost:0.01}" }, { "id": "logic-008", "category": "instruction", "prompt": "Return only the smallest price: $1.25/M, $0.30/M, $2.00/M, $0.14/M." }, { "id": "logic-009", "category": "instruction", "prompt": "Return only the final model ID. Chain: start=gpt-5; replace gpt with glm; append .2; prefix z-ai-." }, { "id": "logic-010", "category": "instruction", "prompt": "Return only the count of unique providers: OpenAI, Anthropic, OpenAI, Google, Z.ai, Google." }, { "id": "pricing-001", "category": "pricing", "prompt": "Return only cheaper or expensive. Model A costs $0.10/M input and $0.30/M output. Model B costs $0.30/M input and $2.50/M output. For any positive input/output workload, Model A is what relative to Model B?" }, { "id": "pricing-002", "category": "pricing", "prompt": "Return only the model name. A got 10 correct for $0.20. B got 8 correct for $0.08. Which has lower dollars per correct answer?" }, { "id": "pricing-003", "category": "pricing", "prompt": "Return only the integer answer. If output tokens are 4x more expensive than input tokens, how many input-token equivalents is 250 output tokens?" }, { "id": "pricing-004", "category": "pricing", "prompt": "Return only JSON. A run used 1000 prompt tokens and 500 completion tokens. Prices are $1/M input and $6/M output. Compute inputCost, outputCost, totalCost in dollars." }, { "id": "pricing-005", "category": "pricing", "prompt": "Return only yes or no. If a model is free for input but charges for output, can a verbose wrong answer still cost money?" }, { "id": "pricing-006", "category": "pricing", "prompt": "Return only the ratio as N:1. Output costs $15/M and input costs $3/M." }, { "id": "pricing-007", "category": "pricing", "prompt": "Return only the integer answer. A $5 pilot budget has already spent $1.75. How many cents remain?" }, { "id": "pricing-008", "category": "pricing", "prompt": "Return only the best metric name: cost per prompt, cost per correct answer, or total tokens. We need to compare accuracy and spend together." } ], "spendUsd": 0.4865153899999998, "monthlySpendAtStartUsd": 2.9986212300000057, "monthlySpendAfterUsd": 3.4851366200000053, "attempted": 686, "correct": 664, "skippedForBudget": 0, "modelSummaries": [ { "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.00657981, "costPerCorrectUsd": 0.00013428183673469389, "promptTokens": 3150, "completionTokens": 5988, "avgCompletionTokens": 122.20408163265306 }, { "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.012822, "costPerCorrectUsd": 0.0002616734693877551, "promptTokens": 3426, "completionTokens": 1566, "avgCompletionTokens": 31.959183673469386 }, { "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.026175000000000007, "costPerCorrectUsd": 0.0005341836734693879, "promptTokens": 3426, "completionTokens": 1174, "avgCompletionTokens": 23.959183673469386 }, { "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.0327486, "costPerCorrectUsd": 0.0006683387755102041, "promptTokens": 3573, "completionTokens": 6306, "avgCompletionTokens": 128.69387755102042 }, { "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.04025999999999998, "costPerCorrectUsd": 0.0008216326530612241, "promptTokens": 3426, "completionTokens": 771, "avgCompletionTokens": 15.73469387755102 }, { "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.06368599999999999, "costPerCorrectUsd": 0.0012997142857142856, "promptTokens": 13216, "completionTokens": 6209, "avgCompletionTokens": 126.71428571428571 }, { "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.09404399999999997, "costPerCorrectUsd": 0.0019192653061224484, "promptTokens": 3086, "completionTokens": 11922, "avgCompletionTokens": 243.30612244897958 }, { "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.043460000000000006, "costPerCorrectUsd": 0.0009054166666666668, "promptTokens": 5232, "completionTokens": 692, "avgCompletionTokens": 14.122448979591837 }, { "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "attempted": 49, "errorCount": 0, "correct": 47, "accuracy": 0.9591836734693877, "spendUsd": 0.021004, "costPerCorrectUsd": 0.00044689361702127654, "promptTokens": 5232, "completionTokens": 1054, "avgCompletionTokens": 21.510204081632654 }, { "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "attempted": 49, "errorCount": 0, "correct": 46, "accuracy": 0.9387755102040817, "spendUsd": 0.0027538999999999992, "costPerCorrectUsd": 0.00005986739130434781, "promptTokens": 3088, "completionTokens": 731, "avgCompletionTokens": 14.918367346938776 }, { "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "attempted": 49, "errorCount": 0, "correct": 46, "accuracy": 0.9387755102040817, "spendUsd": 0.0066560000000000005, "costPerCorrectUsd": 0.00014469565217391304, "promptTokens": 3486, "completionTokens": 634, "avgCompletionTokens": 12.938775510204081 }, { "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "attempted": 49, "errorCount": 0, "correct": 46, "accuracy": 0.9387755102040817, "spendUsd": 0.13232000000000002, "costPerCorrectUsd": 0.002876521739130435, "promptTokens": 5232, "completionTokens": 1600, "avgCompletionTokens": 32.6530612244898 }, { "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "attempted": 49, "errorCount": 0, "correct": 44, "accuracy": 0.8979591836734694, "spendUsd": 0.0016235799999999997, "costPerCorrectUsd": 0.00003689954545454545, "promptTokens": 3075, "completionTokens": 4261, "avgCompletionTokens": 86.95918367346938 }, { "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "attempted": 49, "errorCount": 0, "correct": 44, "accuracy": 0.8979591836734694, "spendUsd": 0.0023825, "costPerCorrectUsd": 0.00005414772727272727, "promptTokens": 3280, "completionTokens": 495, "avgCompletionTokens": 10.10204081632653 } ], "results": [ { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00027, "estimatedUsd": 0.00027, "usage": { "prompt_tokens": 72, "completion_tokens": 33, "total_tokens": 105 }, "latencyMs": 2427 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.000388, "estimatedUsd": 0.000388, "usage": { "prompt_tokens": 88, "completion_tokens": 50, "total_tokens": 138 }, "latencyMs": 2706 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000225, "estimatedUsd": 0.000225, "usage": { "prompt_tokens": 63, "completion_tokens": 27, "total_tokens": 90 }, "latencyMs": 2408 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00024200000000000003, "estimatedUsd": 0.00024200000000000003, "usage": { "prompt_tokens": 62, "completion_tokens": 30, "total_tokens": 92 }, "latencyMs": 2118 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000261, "estimatedUsd": 0.000261, "usage": { "prompt_tokens": 75, "completion_tokens": 31, "total_tokens": 106 }, "latencyMs": 1890 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000098, "estimatedUsd": 0.000098, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 685 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000098, "estimatedUsd": 0.000098, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 672 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000236, "estimatedUsd": 0.000236, "usage": { "prompt_tokens": 68, "completion_tokens": 28, "total_tokens": 96 }, "latencyMs": 1961 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.000445, "estimatedUsd": 0.000445, "usage": { "prompt_tokens": 97, "completion_tokens": 58, "total_tokens": 155 }, "latencyMs": 1149 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.000449, "estimatedUsd": 0.000449, "usage": { "prompt_tokens": 83, "completion_tokens": 61, "total_tokens": 144 }, "latencyMs": 1151 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00010100000000000002, "estimatedUsd": 0.00010100000000000002, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 744 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.000437, "estimatedUsd": 0.000437, "usage": { "prompt_tokens": 83, "completion_tokens": 59, "total_tokens": 142 }, "latencyMs": 2132 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.000249, "estimatedUsd": 0.000249, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 768 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.00037600000000000003, "estimatedUsd": 0.00037600000000000003, "usage": { "prompt_tokens": 70, "completion_tokens": 51, "total_tokens": 121 }, "latencyMs": 1746 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00010100000000000002, "estimatedUsd": 0.00010100000000000002, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 726 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000098, "estimatedUsd": 0.000098, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 781 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00010200000000000001, "estimatedUsd": 0.00010200000000000001, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 2495 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000096, "estimatedUsd": 0.000096, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 582 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000091, "estimatedUsd": 0.000091, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 2339 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000089, "estimatedUsd": 0.000089, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 552 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper extraction succeeded; failed arithmetic favors structured parsing.", "spendUsd": 0.0008979999999999999, "estimatedUsd": 0.0008979999999999999, "usage": { "prompt_tokens": 88, "completion_tokens": 135, "total_tokens": 223 }, "latencyMs": 2245 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000177, "estimatedUsd": 0.000177, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 621 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity can undermine accuracy’s value when token cost compounds.", "spendUsd": 0.000601, "estimatedUsd": 0.000601, "usage": { "prompt_tokens": 79, "completion_tokens": 87, "total_tokens": 166 }, "latencyMs": 4325 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The cost of each correct answer", "spendUsd": 0.000632, "estimatedUsd": 0.000632, "usage": { "prompt_tokens": 74, "completion_tokens": 93, "total_tokens": 167 }, "latencyMs": 3018 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000531, "estimatedUsd": 0.000531, "usage": { "prompt_tokens": 87, "completion_tokens": 74, "total_tokens": 161 }, "latencyMs": 1537 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000107, "estimatedUsd": 0.000107, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 936 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000325, "estimatedUsd": 0.000325, "usage": { "prompt_tokens": 79, "completion_tokens": 41, "total_tokens": 120 }, "latencyMs": 2061 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00011199999999999998, "estimatedUsd": 0.00011199999999999998, "usage": { "prompt_tokens": 70, "completion_tokens": 7, "total_tokens": 77 }, "latencyMs": 792 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.000392, "estimatedUsd": 0.000392, "usage": { "prompt_tokens": 80, "completion_tokens": 52, "total_tokens": 132 }, "latencyMs": 2119 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000232, "estimatedUsd": 0.000232, "usage": { "prompt_tokens": 64, "completion_tokens": 28, "total_tokens": 92 }, "latencyMs": 1536 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00021700000000000002, "estimatedUsd": 0.00021700000000000002, "usage": { "prompt_tokens": 67, "completion_tokens": 25, "total_tokens": 92 }, "latencyMs": 1071 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000228, "estimatedUsd": 0.000228, "usage": { "prompt_tokens": 54, "completion_tokens": 29, "total_tokens": 83 }, "latencyMs": 1803 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00008400000000000001, "estimatedUsd": 0.00008400000000000001, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 748 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000431, "estimatedUsd": 0.000431, "usage": { "prompt_tokens": 47, "completion_tokens": 64, "total_tokens": 111 }, "latencyMs": 2261 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00008400000000000001, "estimatedUsd": 0.00008400000000000001, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 683 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000115, "estimatedUsd": 0.000115, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 1016 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00023700000000000001, "estimatedUsd": 0.00023700000000000001, "usage": { "prompt_tokens": 57, "completion_tokens": 30, "total_tokens": 87 }, "latencyMs": 3093 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000231, "estimatedUsd": 0.000231, "usage": { "prompt_tokens": 57, "completion_tokens": 29, "total_tokens": 86 }, "latencyMs": 2556 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00012, "estimatedUsd": 0.00012, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 639 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000431, "estimatedUsd": 0.000431, "usage": { "prompt_tokens": 65, "completion_tokens": 61, "total_tokens": 126 }, "latencyMs": 1793 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00009, "estimatedUsd": 0.00009, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 2131 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000128, "estimatedUsd": 0.000128, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 708 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00031200000000000005, "estimatedUsd": 0.00031200000000000005, "usage": { "prompt_tokens": 72, "completion_tokens": 40, "total_tokens": 112 }, "latencyMs": 1715 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000234, "estimatedUsd": 0.000234, "usage": { "prompt_tokens": 66, "completion_tokens": 28, "total_tokens": 94 }, "latencyMs": 1641 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.00040199999999999996, "estimatedUsd": 0.00040199999999999996, "usage": { "prompt_tokens": 78, "completion_tokens": 54, "total_tokens": 132 }, "latencyMs": 1832 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00009300000000000001, "estimatedUsd": 0.00009300000000000001, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 942 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00041700000000000005, "estimatedUsd": 0.00041700000000000005, "usage": { "prompt_tokens": 57, "completion_tokens": 60, "total_tokens": 117 }, "latencyMs": 2197 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000246, "estimatedUsd": 0.000246, "usage": { "prompt_tokens": 60, "completion_tokens": 31, "total_tokens": 91 }, "latencyMs": 2278 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000263, "estimatedUsd": 0.000263, "usage": { "prompt_tokens": 65, "completion_tokens": 33, "total_tokens": 98 }, "latencyMs": 3304 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000585, "estimatedUsd": 0.000585, "usage": { "prompt_tokens": 72, "completion_tokens": 27, "total_tokens": 99 }, "latencyMs": 1577 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.000835, "estimatedUsd": 0.000835, "usage": { "prompt_tokens": 88, "completion_tokens": 41, "total_tokens": 129 }, "latencyMs": 2614 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00042750000000000004, "estimatedUsd": 0.00042750000000000004, "usage": { "prompt_tokens": 63, "completion_tokens": 18, "total_tokens": 81 }, "latencyMs": 922 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 1411 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0006225, "estimatedUsd": 0.0006225, "usage": { "prompt_tokens": 75, "completion_tokens": 29, "total_tokens": 104 }, "latencyMs": 1919 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000245, "estimatedUsd": 0.000245, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 680 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000245, "estimatedUsd": 0.000245, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 920 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00048499999999999997, "estimatedUsd": 0.00048499999999999997, "usage": { "prompt_tokens": 68, "completion_tokens": 21, "total_tokens": 89 }, "latencyMs": 1086 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.0007375, "estimatedUsd": 0.0007375, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 996 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.0007325, "estimatedUsd": 0.0007325, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 1105 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0005075, "estimatedUsd": 0.0005075, "usage": { "prompt_tokens": 71, "completion_tokens": 22, "total_tokens": 93 }, "latencyMs": 1087 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0006875, "estimatedUsd": 0.0006875, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 811 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.0008625, "estimatedUsd": 0.0008625, "usage": { "prompt_tokens": 81, "completion_tokens": 44, "total_tokens": 125 }, "latencyMs": 2044 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.000535, "estimatedUsd": 0.000535, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 886 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0002525, "estimatedUsd": 0.0002525, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 805 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000245, "estimatedUsd": 0.000245, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 847 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000255, "estimatedUsd": 0.000255, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 686 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00024, "estimatedUsd": 0.00024, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 894 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0002275, "estimatedUsd": 0.0002275, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 779 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00022250000000000001, "estimatedUsd": 0.00022250000000000001, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 1809 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper extraction succeeded; arithmetic failed, suited structured parsing.", "spendUsd": 0.0015549999999999997, "estimatedUsd": 0.0015549999999999997, "usage": { "prompt_tokens": 88, "completion_tokens": 89, "total_tokens": 177 }, "latencyMs": 1619 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00044249999999999997, "estimatedUsd": 0.00044249999999999997, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 828 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "High verbosity can raise cost despite excellent accuracy.", "spendUsd": 0.0009175, "estimatedUsd": 0.0009175, "usage": { "prompt_tokens": 79, "completion_tokens": 48, "total_tokens": 127 }, "latencyMs": 2553 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost of Each Correct Answer", "spendUsd": 0.00031999999999999997, "estimatedUsd": 0.00031999999999999997, "usage": { "prompt_tokens": 74, "completion_tokens": 9, "total_tokens": 83 }, "latencyMs": 743 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0007725, "estimatedUsd": 0.0007725, "usage": { "prompt_tokens": 87, "completion_tokens": 37, "total_tokens": 124 }, "latencyMs": 828 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0005974999999999999, "estimatedUsd": 0.0005974999999999999, "usage": { "prompt_tokens": 77, "completion_tokens": 27, "total_tokens": 104 }, "latencyMs": 1676 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0007675, "estimatedUsd": 0.0007675, "usage": { "prompt_tokens": 79, "completion_tokens": 38, "total_tokens": 117 }, "latencyMs": 2808 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000565, "estimatedUsd": 0.000565, "usage": { "prompt_tokens": 70, "completion_tokens": 26, "total_tokens": 96 }, "latencyMs": 1189 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.000935, "estimatedUsd": 0.000935, "usage": { "prompt_tokens": 80, "completion_tokens": 49, "total_tokens": 129 }, "latencyMs": 1280 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000535, "estimatedUsd": 0.000535, "usage": { "prompt_tokens": 64, "completion_tokens": 25, "total_tokens": 89 }, "latencyMs": 1317 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0006325, "estimatedUsd": 0.0006325, "usage": { "prompt_tokens": 67, "completion_tokens": 31, "total_tokens": 98 }, "latencyMs": 1864 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00057, "estimatedUsd": 0.00057, "usage": { "prompt_tokens": 54, "completion_tokens": 29, "total_tokens": 83 }, "latencyMs": 2033 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00021, "estimatedUsd": 0.00021, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 646 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0007025, "estimatedUsd": 0.0007025, "usage": { "prompt_tokens": 47, "completion_tokens": 39, "total_tokens": 86 }, "latencyMs": 1197 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00021, "estimatedUsd": 0.00021, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 2085 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0002875, "estimatedUsd": 0.0002875, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 1362 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00045749999999999995, "estimatedUsd": 0.00045749999999999995, "usage": { "prompt_tokens": 57, "completion_tokens": 21, "total_tokens": 78 }, "latencyMs": 1691 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0005175, "estimatedUsd": 0.0005175, "usage": { "prompt_tokens": 57, "completion_tokens": 25, "total_tokens": 82 }, "latencyMs": 1094 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00030000000000000003, "estimatedUsd": 0.00030000000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 1126 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0010025, "estimatedUsd": 0.0010025, "usage": { "prompt_tokens": 65, "completion_tokens": 56, "total_tokens": 121 }, "latencyMs": 1287 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00051, "estimatedUsd": 0.00051, "usage": { "prompt_tokens": 60, "completion_tokens": 24, "total_tokens": 84 }, "latencyMs": 1149 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00032, "estimatedUsd": 0.00032, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 697 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0006450000000000001, "estimatedUsd": 0.0006450000000000001, "usage": { "prompt_tokens": 72, "completion_tokens": 31, "total_tokens": 103 }, "latencyMs": 1918 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000255, "estimatedUsd": 0.000255, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 821 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.001155, "estimatedUsd": 0.001155, "usage": { "prompt_tokens": 78, "completion_tokens": 64, "total_tokens": 142 }, "latencyMs": 1700 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0002325, "estimatedUsd": 0.0002325, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 826 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0005775, "estimatedUsd": 0.0005775, "usage": { "prompt_tokens": 57, "completion_tokens": 29, "total_tokens": 86 }, "latencyMs": 1924 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00051, "estimatedUsd": 0.00051, "usage": { "prompt_tokens": 60, "completion_tokens": 24, "total_tokens": 84 }, "latencyMs": 2177 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0004925, "estimatedUsd": 0.0004925, "usage": { "prompt_tokens": 65, "completion_tokens": 22, "total_tokens": 87 }, "latencyMs": 1452 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00051, "estimatedUsd": 0.00051, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 1741 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0017599999999999998, "estimatedUsd": 0.0017599999999999998, "usage": { "prompt_tokens": 88, "completion_tokens": 44, "total_tokens": 132 }, "latencyMs": 3942 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000495, "estimatedUsd": 0.000495, "usage": { "prompt_tokens": 63, "completion_tokens": 6, "total_tokens": 69 }, "latencyMs": 2511 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00046, "estimatedUsd": 0.00046, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 1071 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000525, "estimatedUsd": 0.000525, "usage": { "prompt_tokens": 75, "completion_tokens": 5, "total_tokens": 80 }, "latencyMs": 1363 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00049, "estimatedUsd": 0.00049, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1154 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00049, "estimatedUsd": 0.00049, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 957 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 68, "completion_tokens": 7, "total_tokens": 75 }, "latencyMs": 1135 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.001475, "estimatedUsd": 0.001475, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 1109 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.001465, "estimatedUsd": 0.001465, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 1987 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000505, "estimatedUsd": 0.000505, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1060 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.001375, "estimatedUsd": 0.001375, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 4606 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.001245, "estimatedUsd": 0.001245, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 3849 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.00107, "estimatedUsd": 0.00107, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 1065 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000505, "estimatedUsd": 0.000505, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 820 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00049, "estimatedUsd": 0.00049, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 2083 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00051, "estimatedUsd": 0.00051, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1490 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00048, "estimatedUsd": 0.00048, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 1020 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000455, "estimatedUsd": 0.000455, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1032 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00044500000000000003, "estimatedUsd": 0.00044500000000000003, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 1153 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper model mastered extraction but failed arithmetic reasoning.", "spendUsd": 0.00428, "estimatedUsd": 0.00428, "usage": { "prompt_tokens": 88, "completion_tokens": 128, "total_tokens": 216 }, "latencyMs": 5111 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0008849999999999999, "estimatedUsd": 0.0008849999999999999, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 1769 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 9, "sentenceCount": 1 }, "output": "High accuracy can lose when output verbosity compounds cost.", "spendUsd": 0.002405, "estimatedUsd": 0.002405, "usage": { "prompt_tokens": 79, "completion_tokens": 67, "total_tokens": 146 }, "latencyMs": 3530 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost per Correct AI Answer", "spendUsd": 0.0006399999999999999, "estimatedUsd": 0.0006399999999999999, "usage": { "prompt_tokens": 74, "completion_tokens": 9, "total_tokens": 83 }, "latencyMs": 1328 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.001545, "estimatedUsd": 0.001545, "usage": { "prompt_tokens": 87, "completion_tokens": 37, "total_tokens": 124 }, "latencyMs": 1876 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0005350000000000001, "estimatedUsd": 0.0005350000000000001, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 1084 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000605, "estimatedUsd": 0.000605, "usage": { "prompt_tokens": 79, "completion_tokens": 7, "total_tokens": 86 }, "latencyMs": 1264 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00131, "estimatedUsd": 0.00131, "usage": { "prompt_tokens": 70, "completion_tokens": 32, "total_tokens": 102 }, "latencyMs": 2144 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0011200000000000001, "estimatedUsd": 0.0011200000000000001, "usage": { "prompt_tokens": 80, "completion_tokens": 24, "total_tokens": 104 }, "latencyMs": 1002 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00059, "estimatedUsd": 0.00059, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 1248 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000485, "estimatedUsd": 0.000485, "usage": { "prompt_tokens": 67, "completion_tokens": 5, "total_tokens": 72 }, "latencyMs": 1345 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0012000000000000001, "estimatedUsd": 0.0012000000000000001, "usage": { "prompt_tokens": 54, "completion_tokens": 31, "total_tokens": 85 }, "latencyMs": 3052 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00042, "estimatedUsd": 0.00042, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 907 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000535, "estimatedUsd": 0.000535, "usage": { "prompt_tokens": 47, "completion_tokens": 10, "total_tokens": 57 }, "latencyMs": 2259 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00042, "estimatedUsd": 0.00042, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 1346 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000575, "estimatedUsd": 0.000575, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 1047 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000435, "estimatedUsd": 0.000435, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 1594 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000435, "estimatedUsd": 0.000435, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 929 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0006000000000000001, "estimatedUsd": 0.0006000000000000001, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 1123 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0006850000000000001, "estimatedUsd": 0.0006850000000000001, "usage": { "prompt_tokens": 65, "completion_tokens": 12, "total_tokens": 77 }, "latencyMs": 1017 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00045000000000000004, "estimatedUsd": 0.00045000000000000004, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 974 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00064, "estimatedUsd": 0.00064, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 855 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00051, "estimatedUsd": 0.00051, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 1365 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00051, "estimatedUsd": 0.00051, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1146 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.00117, "estimatedUsd": 0.00117, "usage": { "prompt_tokens": 78, "completion_tokens": 26, "total_tokens": 104 }, "latencyMs": 996 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000465, "estimatedUsd": 0.000465, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 1217 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000495, "estimatedUsd": 0.000495, "usage": { "prompt_tokens": 57, "completion_tokens": 7, "total_tokens": 64 }, "latencyMs": 1476 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00045000000000000004, "estimatedUsd": 0.00045000000000000004, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 1673 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000565, "estimatedUsd": 0.000565, "usage": { "prompt_tokens": 65, "completion_tokens": 8, "total_tokens": 73 }, "latencyMs": 1197 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00156, "estimatedUsd": 0.00156, "usage": { "prompt_tokens": 106, "completion_tokens": 10, "total_tokens": 116 }, "latencyMs": 4595 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0042, "estimatedUsd": 0.0042, "usage": { "prompt_tokens": 125, "completion_tokens": 59, "total_tokens": 184 }, "latencyMs": 6363 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0011, "estimatedUsd": 0.0011, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 4662 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0010500000000000002, "estimatedUsd": 0.0010500000000000002, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 3629 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0024000000000000002, "estimatedUsd": 0.0024000000000000002, "usage": { "prompt_tokens": 110, "completion_tokens": 26, "total_tokens": 136 }, "latencyMs": 5718 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": null, "detail": "expected 18" }, "output": "", "spendUsd": 0.00113, "estimatedUsd": 0.00113, "usage": { "prompt_tokens": 108, "completion_tokens": 1, "total_tokens": 109 }, "latencyMs": 4504 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00124, "estimatedUsd": 0.00124, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 3611 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00129, "estimatedUsd": 0.00129, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 3297 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.00424, "estimatedUsd": 0.00424, "usage": { "prompt_tokens": 144, "completion_tokens": 56, "total_tokens": 200 }, "latencyMs": 4661 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.005520000000000001, "estimatedUsd": 0.005520000000000001, "usage": { "prompt_tokens": 127, "completion_tokens": 85, "total_tokens": 212 }, "latencyMs": 5882 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00141, "estimatedUsd": 0.00141, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 3064 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true}", "spendUsd": 0.00377, "estimatedUsd": 0.00377, "usage": { "prompt_tokens": 132, "completion_tokens": 49, "total_tokens": 181 }, "latencyMs": 3542 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0028200000000000005, "estimatedUsd": 0.0028200000000000005, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 3572 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.00477, "estimatedUsd": 0.00477, "usage": { "prompt_tokens": 107, "completion_tokens": 74, "total_tokens": 181 }, "latencyMs": 4553 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 3646 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00129, "estimatedUsd": 0.00129, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 3526 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0012399999999999998, "estimatedUsd": 0.0012399999999999998, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 3827 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 3846 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00115, "estimatedUsd": 0.00115, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 4877 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00107, "estimatedUsd": 0.00107, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 3934 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded extraction but failed arithmetic reasoning.", "spendUsd": 0.0088, "estimatedUsd": 0.0088, "usage": { "prompt_tokens": 150, "completion_tokens": 146, "total_tokens": 296 }, "latencyMs": 5441 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0026, "estimatedUsd": 0.0026, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 3588 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.0021, "estimatedUsd": 0.0021, "usage": { "prompt_tokens": 125, "completion_tokens": 17, "total_tokens": 142 }, "latencyMs": 3554 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost Per Correct Answer in AI", "spendUsd": 0.0019399999999999999, "estimatedUsd": 0.0019399999999999999, "usage": { "prompt_tokens": 114, "completion_tokens": 16, "total_tokens": 130 }, "latencyMs": 3660 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00639, "estimatedUsd": 0.00639, "usage": { "prompt_tokens": 149, "completion_tokens": 98, "total_tokens": 247 }, "latencyMs": 5421 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00176, "estimatedUsd": 0.00176, "usage": { "prompt_tokens": 116, "completion_tokens": 12, "total_tokens": 128 }, "latencyMs": 4404 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.01963, "estimatedUsd": 0.01963, "usage": { "prompt_tokens": 108, "completion_tokens": 371, "total_tokens": 479 }, "latencyMs": 14229 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00143, "estimatedUsd": 0.00143, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 3337 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00434, "estimatedUsd": 0.00434, "usage": { "prompt_tokens": 124, "completion_tokens": 62, "total_tokens": 186 }, "latencyMs": 6076 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00132, "estimatedUsd": 0.00132, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 3906 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected fetch" }, "output": "", "spendUsd": 0.00104, "estimatedUsd": 0.00104, "usage": { "prompt_tokens": 99, "completion_tokens": 1, "total_tokens": 100 }, "latencyMs": 3422 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0026, "estimatedUsd": 0.0026, "usage": { "prompt_tokens": 85, "completion_tokens": 35, "total_tokens": 120 }, "latencyMs": 5133 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00091, "estimatedUsd": 0.00091, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 3203 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0040999999999999995, "estimatedUsd": 0.0040999999999999995, "usage": { "prompt_tokens": 70, "completion_tokens": 68, "total_tokens": 138 }, "latencyMs": 5095 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 76, "completion_tokens": 12, "total_tokens": 88 }, "latencyMs": 4459 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00141, "estimatedUsd": 0.00141, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 3159 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0010600000000000002, "estimatedUsd": 0.0010600000000000002, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 3538 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0011, "estimatedUsd": 0.0011, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 3022 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00132, "estimatedUsd": 0.00132, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 3558 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0033299999999999996, "estimatedUsd": 0.0033299999999999996, "usage": { "prompt_tokens": 98, "completion_tokens": 47, "total_tokens": 145 }, "latencyMs": 5123 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0025599999999999998, "estimatedUsd": 0.0025599999999999998, "usage": { "prompt_tokens": 96, "completion_tokens": 32, "total_tokens": 128 }, "latencyMs": 4311 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0016100000000000003, "estimatedUsd": 0.0016100000000000003, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 3553 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00302, "estimatedUsd": 0.00302, "usage": { "prompt_tokens": 102, "completion_tokens": 40, "total_tokens": 142 }, "latencyMs": 5119 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0012100000000000001, "estimatedUsd": 0.0012100000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 3169 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.00493, "estimatedUsd": 0.00493, "usage": { "prompt_tokens": 123, "completion_tokens": 74, "total_tokens": 197 }, "latencyMs": 5411 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0011099999999999999, "estimatedUsd": 0.0011099999999999999, "usage": { "prompt_tokens": 91, "completion_tokens": 4, "total_tokens": 95 }, "latencyMs": 3269 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 5:1" }, "output": "15:5 simplifies to 5:1 5:1", "spendUsd": 0.00179, "estimatedUsd": 0.00179, "usage": { "prompt_tokens": 84, "completion_tokens": 19, "total_tokens": 103 }, "latencyMs": 3512 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00104, "estimatedUsd": 0.00104, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 3648 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00254, "estimatedUsd": 0.00254, "usage": { "prompt_tokens": 99, "completion_tokens": 31, "total_tokens": 130 }, "latencyMs": 5320 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000605, "estimatedUsd": 0.000605, "usage": { "prompt_tokens": 106, "completion_tokens": 3, "total_tokens": 109 }, "latencyMs": 3280 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0007, "estimatedUsd": 0.0007, "usage": { "prompt_tokens": 125, "completion_tokens": 3, "total_tokens": 128 }, "latencyMs": 2006 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 2356 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0005250000000000001, "estimatedUsd": 0.0005250000000000001, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 1595 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000625, "estimatedUsd": 0.000625, "usage": { "prompt_tokens": 110, "completion_tokens": 3, "total_tokens": 113 }, "latencyMs": 1527 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000615, "estimatedUsd": 0.000615, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 1599 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00062, "estimatedUsd": 0.00062, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 4983 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 1552 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.002245, "estimatedUsd": 0.002245, "usage": { "prompt_tokens": 144, "completion_tokens": 61, "total_tokens": 205 }, "latencyMs": 4958 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.002235, "estimatedUsd": 0.002235, "usage": { "prompt_tokens": 127, "completion_tokens": 64, "total_tokens": 191 }, "latencyMs": 2132 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000705, "estimatedUsd": 0.000705, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 1245 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0022849999999999997, "estimatedUsd": 0.0022849999999999997, "usage": { "prompt_tokens": 132, "completion_tokens": 65, "total_tokens": 197 }, "latencyMs": 1717 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0014100000000000002, "estimatedUsd": 0.0014100000000000002, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 2347 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0017599999999999998, "estimatedUsd": 0.0017599999999999998, "usage": { "prompt_tokens": 107, "completion_tokens": 49, "total_tokens": 156 }, "latencyMs": 2617 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 2117 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 1680 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0006199999999999999, "estimatedUsd": 0.0006199999999999999, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 1436 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 1581 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000575, "estimatedUsd": 0.000575, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 1537 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000535, "estimatedUsd": 0.000535, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 1364 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model excelled extraction, failed multi-step arithmetic reasoning.", "spendUsd": 0.0015249999999999999, "estimatedUsd": 0.0015249999999999999, "usage": { "prompt_tokens": 150, "completion_tokens": 31, "total_tokens": 181 }, "latencyMs": 2378 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0013, "estimatedUsd": 0.0013, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 1563 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Verbosity increases cost per correct answer.", "spendUsd": 0.00105, "estimatedUsd": 0.00105, "usage": { "prompt_tokens": 125, "completion_tokens": 17, "total_tokens": 142 }, "latencyMs": 1758 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost Per Correct Answer", "spendUsd": 0.000895, "estimatedUsd": 0.000895, "usage": { "prompt_tokens": 114, "completion_tokens": 13, "total_tokens": 127 }, "latencyMs": 1770 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00227, "estimatedUsd": 0.00227, "usage": { "prompt_tokens": 149, "completion_tokens": 61, "total_tokens": 210 }, "latencyMs": 2258 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000655, "estimatedUsd": 0.000655, "usage": { "prompt_tokens": 116, "completion_tokens": 3, "total_tokens": 119 }, "latencyMs": 1243 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000665, "estimatedUsd": 0.000665, "usage": { "prompt_tokens": 108, "completion_tokens": 5, "total_tokens": 113 }, "latencyMs": 1550 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000715, "estimatedUsd": 0.000715, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 2185 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.0017949999999999997, "estimatedUsd": 0.0017949999999999997, "usage": { "prompt_tokens": 124, "completion_tokens": 47, "total_tokens": 171 }, "latencyMs": 1931 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 1897 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00062, "estimatedUsd": 0.00062, "usage": { "prompt_tokens": 99, "completion_tokens": 5, "total_tokens": 104 }, "latencyMs": 1393 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 1461 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000455, "estimatedUsd": 0.000455, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 1589 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000575, "estimatedUsd": 0.000575, "usage": { "prompt_tokens": 70, "completion_tokens": 9, "total_tokens": 79 }, "latencyMs": 2005 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0005300000000000001, "estimatedUsd": 0.0005300000000000001, "usage": { "prompt_tokens": 76, "completion_tokens": 6, "total_tokens": 82 }, "latencyMs": 2854 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000705, "estimatedUsd": 0.000705, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 3228 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0005300000000000001, "estimatedUsd": 0.0005300000000000001, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 1933 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 5915 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 1799 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000815, "estimatedUsd": 0.000815, "usage": { "prompt_tokens": 98, "completion_tokens": 13, "total_tokens": 111 }, "latencyMs": 2156 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000555, "estimatedUsd": 0.000555, "usage": { "prompt_tokens": 96, "completion_tokens": 3, "total_tokens": 99 }, "latencyMs": 2136 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0008050000000000002, "estimatedUsd": 0.0008050000000000002, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 1463 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000585, "estimatedUsd": 0.000585, "usage": { "prompt_tokens": 102, "completion_tokens": 3, "total_tokens": 105 }, "latencyMs": 1407 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0006050000000000001, "estimatedUsd": 0.0006050000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 1116 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.001515, "estimatedUsd": 0.001515, "usage": { "prompt_tokens": 123, "completion_tokens": 36, "total_tokens": 159 }, "latencyMs": 1852 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.00058, "estimatedUsd": 0.00058, "usage": { "prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96 }, "latencyMs": 1285 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 5:1" }, "output": "15:3 simplified is 5:1.", "spendUsd": 0.0007949999999999999, "estimatedUsd": 0.0007949999999999999, "usage": { "prompt_tokens": 84, "completion_tokens": 15, "total_tokens": 99 }, "latencyMs": 1446 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00052, "estimatedUsd": 0.00052, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 1640 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0007199999999999999, "estimatedUsd": 0.0007199999999999999, "usage": { "prompt_tokens": 99, "completion_tokens": 9, "total_tokens": 108 }, "latencyMs": 1219 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00045200000000000004, "estimatedUsd": 0.00045200000000000004, "usage": { "prompt_tokens": 106, "completion_tokens": 24, "total_tokens": 130 }, "latencyMs": 3740 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00102, "estimatedUsd": 0.00102, "usage": { "prompt_tokens": 125, "completion_tokens": 77, "total_tokens": 202 }, "latencyMs": 3419 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00022, "estimatedUsd": 0.00022, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 2623 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00021, "estimatedUsd": 0.00021, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 2955 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00025, "estimatedUsd": 0.00025, "usage": { "prompt_tokens": 110, "completion_tokens": 3, "total_tokens": 113 }, "latencyMs": 2285 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000246, "estimatedUsd": 0.000246, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 2524 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000248, "estimatedUsd": 0.000248, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 2532 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000258, "estimatedUsd": 0.000258, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 2586 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0008979999999999999, "estimatedUsd": 0.0008979999999999999, "usage": { "prompt_tokens": 144, "completion_tokens": 61, "total_tokens": 205 }, "latencyMs": 3132 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0008939999999999999, "estimatedUsd": 0.0008939999999999999, "usage": { "prompt_tokens": 127, "completion_tokens": 64, "total_tokens": 191 }, "latencyMs": 2470 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000282, "estimatedUsd": 0.000282, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 3103 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.000914, "estimatedUsd": 0.000914, "usage": { "prompt_tokens": 132, "completion_tokens": 65, "total_tokens": 197 }, "latencyMs": 3037 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.000564, "estimatedUsd": 0.000564, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 2042 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.000584, "estimatedUsd": 0.000584, "usage": { "prompt_tokens": 107, "completion_tokens": 37, "total_tokens": 144 }, "latencyMs": 2247 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000272, "estimatedUsd": 0.000272, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 2437 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000258, "estimatedUsd": 0.000258, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 2685 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00024799999999999996, "estimatedUsd": 0.00024799999999999996, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 2431 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000272, "estimatedUsd": 0.000272, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 3082 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.00024, "estimatedUsd": 0.00024, "usage": { "prompt_tokens": 95, "completion_tokens": 5, "total_tokens": 100 }, "latencyMs": 2122 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.000214, "estimatedUsd": 0.000214, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 2304 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded at extraction, failed multi-step arithmetic.", "spendUsd": 0.00059, "estimatedUsd": 0.00059, "usage": { "prompt_tokens": 150, "completion_tokens": 29, "total_tokens": 179 }, "latencyMs": 2542 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00052, "estimatedUsd": 0.00052, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 3893 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 9, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer despite accuracy.", "spendUsd": 0.00048, "estimatedUsd": 0.00048, "usage": { "prompt_tokens": 125, "completion_tokens": 23, "total_tokens": 148 }, "latencyMs": 2840 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost Per Correct Answer Comparison", "spendUsd": 0.000398, "estimatedUsd": 0.000398, "usage": { "prompt_tokens": 114, "completion_tokens": 17, "total_tokens": 131 }, "latencyMs": 2345 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000908, "estimatedUsd": 0.000908, "usage": { "prompt_tokens": 149, "completion_tokens": 61, "total_tokens": 210 }, "latencyMs": 2559 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000262, "estimatedUsd": 0.000262, "usage": { "prompt_tokens": 116, "completion_tokens": 3, "total_tokens": 119 }, "latencyMs": 2262 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0006559999999999999, "estimatedUsd": 0.0006559999999999999, "usage": { "prompt_tokens": 108, "completion_tokens": 44, "total_tokens": 152 }, "latencyMs": 2828 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000286, "estimatedUsd": 0.000286, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 2239 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.001108, "estimatedUsd": 0.001108, "usage": { "prompt_tokens": 124, "completion_tokens": 86, "total_tokens": 210 }, "latencyMs": 2873 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000264, "estimatedUsd": 0.000264, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 2263 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000248, "estimatedUsd": 0.000248, "usage": { "prompt_tokens": 99, "completion_tokens": 5, "total_tokens": 104 }, "latencyMs": 3335 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Grok", "spendUsd": 0.00022, "estimatedUsd": 0.00022, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 2521 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000182, "estimatedUsd": 0.000182, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 2235 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 70, "completion_tokens": 9, "total_tokens": 79 }, "latencyMs": 2836 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000212, "estimatedUsd": 0.000212, "usage": { "prompt_tokens": 76, "completion_tokens": 6, "total_tokens": 82 }, "latencyMs": 2313 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000282, "estimatedUsd": 0.000282, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 2249 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000212, "estimatedUsd": 0.000212, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 3293 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00039, "estimatedUsd": 0.00039, "usage": { "prompt_tokens": 85, "completion_tokens": 22, "total_tokens": 107 }, "latencyMs": 2753 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000264, "estimatedUsd": 0.000264, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 2323 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000956, "estimatedUsd": 0.000956, "usage": { "prompt_tokens": 98, "completion_tokens": 76, "total_tokens": 174 }, "latencyMs": 2753 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000222, "estimatedUsd": 0.000222, "usage": { "prompt_tokens": 96, "completion_tokens": 3, "total_tokens": 99 }, "latencyMs": 2094 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000322, "estimatedUsd": 0.000322, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 2332 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000754, "estimatedUsd": 0.000754, "usage": { "prompt_tokens": 102, "completion_tokens": 55, "total_tokens": 157 }, "latencyMs": 2878 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000242, "estimatedUsd": 0.000242, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 3054 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.001206, "estimatedUsd": 0.001206, "usage": { "prompt_tokens": 123, "completion_tokens": 96, "total_tokens": 219 }, "latencyMs": 3610 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000222, "estimatedUsd": 0.000222, "usage": { "prompt_tokens": 91, "completion_tokens": 4, "total_tokens": 95 }, "latencyMs": 2498 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 5:1" }, "output": "15:3, simplified: **5:1**", "spendUsd": 0.00031800000000000003, "estimatedUsd": 0.00031800000000000003, "usage": { "prompt_tokens": 84, "completion_tokens": 15, "total_tokens": 99 }, "latencyMs": 2613 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000208, "estimatedUsd": 0.000208, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 2883 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "Cost per correct answer", "spendUsd": 0.000298, "estimatedUsd": 0.000298, "usage": { "prompt_tokens": 99, "completion_tokens": 10, "total_tokens": 109 }, "latencyMs": 2710 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000099, "estimatedUsd": 0.000099, "usage": { "prompt_tokens": 74, "completion_tokens": 5, "total_tokens": 79 }, "latencyMs": 936 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 6, "detail": "expected 8" }, "output": "6", "spendUsd": 0.000115, "estimatedUsd": 0.000115, "usage": { "prompt_tokens": 90, "completion_tokens": 5, "total_tokens": 95 }, "latencyMs": 1275 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 62, "completion_tokens": 6, "total_tokens": 68 }, "latencyMs": 1059 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000087, "estimatedUsd": 0.000087, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 3508 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 18, "detail": "expected 20" }, "output": "18", "spendUsd": 0.00009999999999999999, "estimatedUsd": 0.00009999999999999999, "usage": { "prompt_tokens": 75, "completion_tokens": 5, "total_tokens": 80 }, "latencyMs": 1281 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00009499999999999999, "estimatedUsd": 0.00009499999999999999, "usage": { "prompt_tokens": 70, "completion_tokens": 5, "total_tokens": 75 }, "latencyMs": 1243 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000093, "estimatedUsd": 0.000093, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1141 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000104, "estimatedUsd": 0.000104, "usage": { "prompt_tokens": 69, "completion_tokens": 7, "total_tokens": 76 }, "latencyMs": 1384 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.000375, "estimatedUsd": 0.000375, "usage": { "prompt_tokens": 100, "completion_tokens": 55, "total_tokens": 155 }, "latencyMs": 1346 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.000371, "estimatedUsd": 0.000371, "usage": { "prompt_tokens": 86, "completion_tokens": 57, "total_tokens": 143 }, "latencyMs": 884 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000094, "estimatedUsd": 0.000094, "usage": { "prompt_tokens": 74, "completion_tokens": 4, "total_tokens": 78 }, "latencyMs": 852 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.00038700000000000003, "estimatedUsd": 0.00038700000000000003, "usage": { "prompt_tokens": 92, "completion_tokens": 59, "total_tokens": 151 }, "latencyMs": 931 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00021700000000000002, "estimatedUsd": 0.00021700000000000002, "usage": { "prompt_tokens": 82, "completion_tokens": 27, "total_tokens": 109 }, "latencyMs": 1005 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.00029299999999999997, "estimatedUsd": 0.00029299999999999997, "usage": { "prompt_tokens": 73, "completion_tokens": 44, "total_tokens": 117 }, "latencyMs": 1067 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000094, "estimatedUsd": 0.000094, "usage": { "prompt_tokens": 74, "completion_tokens": 4, "total_tokens": 78 }, "latencyMs": 924 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000091, "estimatedUsd": 0.000091, "usage": { "prompt_tokens": 71, "completion_tokens": 4, "total_tokens": 75 }, "latencyMs": 1276 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000093, "estimatedUsd": 0.000093, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 997 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000089, "estimatedUsd": 0.000089, "usage": { "prompt_tokens": 69, "completion_tokens": 4, "total_tokens": 73 }, "latencyMs": 2255 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00007999999999999999, "estimatedUsd": 0.00007999999999999999, "usage": { "prompt_tokens": 60, "completion_tokens": 4, "total_tokens": 64 }, "latencyMs": 1092 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.000078, "estimatedUsd": 0.000078, "usage": { "prompt_tokens": 58, "completion_tokens": 4, "total_tokens": 62 }, "latencyMs": 1305 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper extraction model failed complex arithmetic reasoning tasks.", "spendUsd": 0.000154, "estimatedUsd": 0.000154, "usage": { "prompt_tokens": 89, "completion_tokens": 13, "total_tokens": 102 }, "latencyMs": 1374 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000165, "estimatedUsd": 0.000165, "usage": { "prompt_tokens": 95, "completion_tokens": 14, "total_tokens": 109 }, "latencyMs": 1294 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High output verbosity increases cost per correct answer despite accuracy.", "spendUsd": 0.000156, "estimatedUsd": 0.000156, "usage": { "prompt_tokens": 81, "completion_tokens": 15, "total_tokens": 96 }, "latencyMs": 1644 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost to Correct AI Benchmark Answers", "spendUsd": 0.000134, "estimatedUsd": 0.000134, "usage": { "prompt_tokens": 74, "completion_tokens": 12, "total_tokens": 86 }, "latencyMs": 1324 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00028399999999999996, "estimatedUsd": 0.00028399999999999996, "usage": { "prompt_tokens": 89, "completion_tokens": 39, "total_tokens": 128 }, "latencyMs": 1469 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000106, "estimatedUsd": 0.000106, "usage": { "prompt_tokens": 81, "completion_tokens": 5, "total_tokens": 86 }, "latencyMs": 1287 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000115, "estimatedUsd": 0.000115, "usage": { "prompt_tokens": 80, "completion_tokens": 7, "total_tokens": 87 }, "latencyMs": 3244 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00009999999999999999, "estimatedUsd": 0.00009999999999999999, "usage": { "prompt_tokens": 70, "completion_tokens": 6, "total_tokens": 76 }, "latencyMs": 1335 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.000299, "estimatedUsd": 0.000299, "usage": { "prompt_tokens": 84, "completion_tokens": 43, "total_tokens": 127 }, "latencyMs": 994 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00010899999999999999, "estimatedUsd": 0.00010899999999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 1339 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000088, "estimatedUsd": 0.000088, "usage": { "prompt_tokens": 68, "completion_tokens": 4, "total_tokens": 72 }, "latencyMs": 2767 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Sonnet", "spendUsd": 0.000084, "estimatedUsd": 0.000084, "usage": { "prompt_tokens": 54, "completion_tokens": 6, "total_tokens": 60 }, "latencyMs": 1031 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00007199999999999999, "estimatedUsd": 0.00007199999999999999, "usage": { "prompt_tokens": 52, "completion_tokens": 4, "total_tokens": 56 }, "latencyMs": 2399 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000096, "estimatedUsd": 0.000096, "usage": { "prompt_tokens": 46, "completion_tokens": 10, "total_tokens": 56 }, "latencyMs": 2275 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000077, "estimatedUsd": 0.000077, "usage": { "prompt_tokens": 47, "completion_tokens": 6, "total_tokens": 53 }, "latencyMs": 1620 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00010899999999999999, "estimatedUsd": 0.00010899999999999999, "usage": { "prompt_tokens": 69, "completion_tokens": 8, "total_tokens": 77 }, "latencyMs": 1131 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00007500000000000001, "estimatedUsd": 0.00007500000000000001, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 1157 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00007500000000000001, "estimatedUsd": 0.00007500000000000001, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 1207 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00011300000000000001, "estimatedUsd": 0.00011300000000000001, "usage": { "prompt_tokens": 68, "completion_tokens": 9, "total_tokens": 77 }, "latencyMs": 1699 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00013099999999999999, "estimatedUsd": 0.00013099999999999999, "usage": { "prompt_tokens": 66, "completion_tokens": 13, "total_tokens": 79 }, "latencyMs": 990 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000087, "estimatedUsd": 0.000087, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 932 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000116, "estimatedUsd": 0.000116, "usage": { "prompt_tokens": 96, "completion_tokens": 4, "total_tokens": 100 }, "latencyMs": 1005 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 72, "completion_tokens": 4, "total_tokens": 76 }, "latencyMs": 890 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000097, "estimatedUsd": 0.000097, "usage": { "prompt_tokens": 67, "completion_tokens": 6, "total_tokens": 73 }, "latencyMs": 1634 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.000315, "estimatedUsd": 0.000315, "usage": { "prompt_tokens": 85, "completion_tokens": 46, "total_tokens": 131 }, "latencyMs": 1188 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes.", "spendUsd": 0.000086, "estimatedUsd": 0.000086, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 835 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 57, "completion_tokens": 7, "total_tokens": 64 }, "latencyMs": 1135 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000084, "estimatedUsd": 0.000084, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 939 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000098, "estimatedUsd": 0.000098, "usage": { "prompt_tokens": 63, "completion_tokens": 7, "total_tokens": 70 }, "latencyMs": 1195 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0007965, "estimatedUsd": 0.0007965, "usage": { "prompt_tokens": 66, "completion_tokens": 93, "total_tokens": 159 }, "latencyMs": 1224 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.001743, "estimatedUsd": 0.001743, "usage": { "prompt_tokens": 87, "completion_tokens": 215, "total_tokens": 302 }, "latencyMs": 1554 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0015945000000000002, "estimatedUsd": 0.0015945000000000002, "usage": { "prompt_tokens": 58, "completion_tokens": 201, "total_tokens": 259 }, "latencyMs": 1550 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0009615, "estimatedUsd": 0.0009615, "usage": { "prompt_tokens": 56, "completion_tokens": 117, "total_tokens": 173 }, "latencyMs": 1320 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.001407, "estimatedUsd": 0.001407, "usage": { "prompt_tokens": 68, "completion_tokens": 174, "total_tokens": 242 }, "latencyMs": 1357 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.001104, "estimatedUsd": 0.001104, "usage": { "prompt_tokens": 61, "completion_tokens": 135, "total_tokens": 196 }, "latencyMs": 1750 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00078, "estimatedUsd": 0.00078, "usage": { "prompt_tokens": 60, "completion_tokens": 92, "total_tokens": 152 }, "latencyMs": 1276 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.0006405, "estimatedUsd": 0.0006405, "usage": { "prompt_tokens": 62, "completion_tokens": 73, "total_tokens": 135 }, "latencyMs": 1036 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.3, \"outputPerM\": 2.5, \"context\": 1048576}", "spendUsd": 0.0037080000000000004, "estimatedUsd": 0.0037080000000000004, "usage": { "prompt_tokens": 97, "completion_tokens": 475, "total_tokens": 572 }, "latencyMs": 2554 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0076815, "estimatedUsd": 0.0076815, "usage": { "prompt_tokens": 76, "completion_tokens": 1009, "total_tokens": 1085 }, "latencyMs": 5930 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0013695, "estimatedUsd": 0.0013695, "usage": { "prompt_tokens": 63, "completion_tokens": 170, "total_tokens": 233 }, "latencyMs": 1388 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true}", "spendUsd": 0.003294, "estimatedUsd": 0.003294, "usage": { "prompt_tokens": 86, "completion_tokens": 422, "total_tokens": 508 }, "latencyMs": 1897 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0031125000000000002, "estimatedUsd": 0.0031125000000000002, "usage": { "prompt_tokens": 75, "completion_tokens": 400, "total_tokens": 475 }, "latencyMs": 2185 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.001668, "estimatedUsd": 0.001668, "usage": { "prompt_tokens": 62, "completion_tokens": 210, "total_tokens": 272 }, "latencyMs": 1733 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0006165, "estimatedUsd": 0.0006165, "usage": { "prompt_tokens": 61, "completion_tokens": 70, "total_tokens": 131 }, "latencyMs": 857 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0007139999999999999, "estimatedUsd": 0.0007139999999999999, "usage": { "prompt_tokens": 56, "completion_tokens": 84, "total_tokens": 140 }, "latencyMs": 1698 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0015480000000000001, "estimatedUsd": 0.0015480000000000001, "usage": { "prompt_tokens": 57, "completion_tokens": 195, "total_tokens": 252 }, "latencyMs": 1642 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0006960000000000001, "estimatedUsd": 0.0006960000000000001, "usage": { "prompt_tokens": 59, "completion_tokens": 81, "total_tokens": 140 }, "latencyMs": 1137 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0009945, "estimatedUsd": 0.0009945, "usage": { "prompt_tokens": 53, "completion_tokens": 122, "total_tokens": 175 }, "latencyMs": 1464 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0005685, "estimatedUsd": 0.0005685, "usage": { "prompt_tokens": 49, "completion_tokens": 66, "total_tokens": 115 }, "latencyMs": 2728 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "The cheaper model passed extraction but failed arithmetic.", "spendUsd": 0.006390000000000001, "estimatedUsd": 0.006390000000000001, "usage": { "prompt_tokens": 80, "completion_tokens": 836, "total_tokens": 916 }, "latencyMs": 4731 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.001557, "estimatedUsd": 0.001557, "usage": { "prompt_tokens": 83, "completion_tokens": 191, "total_tokens": 274 }, "latencyMs": 1503 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "High verbosity increases overall cost.", "spendUsd": 0.0036810000000000002, "estimatedUsd": 0.0036810000000000002, "usage": { "prompt_tokens": 74, "completion_tokens": 476, "total_tokens": 550 }, "latencyMs": 2800 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "AI Cost Per Correct Answer", "spendUsd": 0.0053415, "estimatedUsd": 0.0053415, "usage": { "prompt_tokens": 66, "completion_tokens": 699, "total_tokens": 765 }, "latencyMs": 3737 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0029535000000000004, "estimatedUsd": 0.0029535000000000004, "usage": { "prompt_tokens": 79, "completion_tokens": 378, "total_tokens": 457 }, "latencyMs": 1835 }, { "taskId": "code-001", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0025185, "estimatedUsd": 0.0025185, "usage": { "prompt_tokens": 74, "completion_tokens": 321, "total_tokens": 395 }, "latencyMs": 1767 }, { "taskId": "code-002", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.001227, "estimatedUsd": 0.001227, "usage": { "prompt_tokens": 68, "completion_tokens": 150, "total_tokens": 218 }, "latencyMs": 1058 }, { "taskId": "code-003", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.002292, "estimatedUsd": 0.002292, "usage": { "prompt_tokens": 63, "completion_tokens": 293, "total_tokens": 356 }, "latencyMs": 2561 }, { "taskId": "code-004", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03}", "spendUsd": 0.0026445, "estimatedUsd": 0.0026445, "usage": { "prompt_tokens": 78, "completion_tokens": 337, "total_tokens": 415 }, "latencyMs": 2064 }, { "taskId": "code-005", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0018030000000000001, "estimatedUsd": 0.0018030000000000001, "usage": { "prompt_tokens": 57, "completion_tokens": 229, "total_tokens": 286 }, "latencyMs": 1576 }, { "taskId": "code-006", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000789, "estimatedUsd": 0.000789, "usage": { "prompt_tokens": 61, "completion_tokens": 93, "total_tokens": 154 }, "latencyMs": 1295 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000687, "estimatedUsd": 0.000687, "usage": { "prompt_tokens": 43, "completion_tokens": 83, "total_tokens": 126 }, "latencyMs": 1600 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0012465, "estimatedUsd": 0.0012465, "usage": { "prompt_tokens": 46, "completion_tokens": 157, "total_tokens": 203 }, "latencyMs": 1836 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0022530000000000002, "estimatedUsd": 0.0022530000000000002, "usage": { "prompt_tokens": 37, "completion_tokens": 293, "total_tokens": 330 }, "latencyMs": 1856 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.001125, "estimatedUsd": 0.001125, "usage": { "prompt_tokens": 40, "completion_tokens": 142, "total_tokens": 182 }, "latencyMs": 1180 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0017970000000000002, "estimatedUsd": 0.0017970000000000002, "usage": { "prompt_tokens": 58, "completion_tokens": 228, "total_tokens": 286 }, "latencyMs": 1682 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0011355, "estimatedUsd": 0.0011355, "usage": { "prompt_tokens": 47, "completion_tokens": 142, "total_tokens": 189 }, "latencyMs": 1065 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.001128, "estimatedUsd": 0.001128, "usage": { "prompt_tokens": 47, "completion_tokens": 141, "total_tokens": 188 }, "latencyMs": 1580 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.001701, "estimatedUsd": 0.001701, "usage": { "prompt_tokens": 64, "completion_tokens": 214, "total_tokens": 278 }, "latencyMs": 1512 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0008519999999999999, "estimatedUsd": 0.0008519999999999999, "usage": { "prompt_tokens": 58, "completion_tokens": 102, "total_tokens": 160 }, "latencyMs": 1057 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0009015, "estimatedUsd": 0.0009015, "usage": { "prompt_tokens": 51, "completion_tokens": 110, "total_tokens": 161 }, "latencyMs": 1258 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0020145, "estimatedUsd": 0.0020145, "usage": { "prompt_tokens": 93, "completion_tokens": 250, "total_tokens": 343 }, "latencyMs": 2019 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0020925, "estimatedUsd": 0.0020925, "usage": { "prompt_tokens": 65, "completion_tokens": 266, "total_tokens": 331 }, "latencyMs": 2189 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0012360000000000001, "estimatedUsd": 0.0012360000000000001, "usage": { "prompt_tokens": 59, "completion_tokens": 153, "total_tokens": 212 }, "latencyMs": 1285 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.003921, "estimatedUsd": 0.003921, "usage": { "prompt_tokens": 74, "completion_tokens": 508, "total_tokens": 582 }, "latencyMs": 2462 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.0018945, "estimatedUsd": 0.0018945, "usage": { "prompt_tokens": 53, "completion_tokens": 242, "total_tokens": 295 }, "latencyMs": 1738 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0012225, "estimatedUsd": 0.0012225, "usage": { "prompt_tokens": 50, "completion_tokens": 153, "total_tokens": 203 }, "latencyMs": 1548 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0007289999999999999, "estimatedUsd": 0.0007289999999999999, "usage": { "prompt_tokens": 51, "completion_tokens": 87, "total_tokens": 138 }, "latencyMs": 1307 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0019125000000000001, "estimatedUsd": 0.0019125000000000001, "usage": { "prompt_tokens": 55, "completion_tokens": 244, "total_tokens": 299 }, "latencyMs": 2100 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0000267, "estimatedUsd": 0.0000267, "usage": { "prompt_tokens": 64, "completion_tokens": 3, "total_tokens": 67 }, "latencyMs": 461 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "actual": null, "detail": "expected 8" }, "output": "To find the total cost in cents: 1. **Input tokens cost:** - Rate: $0.30 per 1M tokens = $0.30 / 1,000,000 = $0.0000003 per token. - For 100,000 tokens: 100,000 × $0.0000003 = $0.03. 2. **Output tokens cost:** - Rate: $2.50 per 1M tokens = $2.50 / 1,000,000 = $0.0000025 per token. - For 20,000 tokens: 20,000 × $0.0000025 = $0.05. 3. **Total cost in dollars:** - $0.03 + $0.05 = $0.08. 4. **Convert to cents:** - $0.08 × 100 = 8 cents. 8", "spendUsd": 0.0006567000000000001, "estimatedUsd": 0.0006567000000000001, "usage": { "prompt_tokens": 89, "completion_tokens": 252, "total_tokens": 341 }, "latencyMs": 927 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000029900000000000002, "estimatedUsd": 0.000029900000000000002, "usage": { "prompt_tokens": 58, "completion_tokens": 5, "total_tokens": 63 }, "latencyMs": 380 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000021199999999999997, "estimatedUsd": 0.000021199999999999997, "usage": { "prompt_tokens": 54, "completion_tokens": 2, "total_tokens": 56 }, "latencyMs": 922 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000025399999999999997, "estimatedUsd": 0.000025399999999999997, "usage": { "prompt_tokens": 68, "completion_tokens": 2, "total_tokens": 70 }, "latencyMs": 1057 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000023299999999999997, "estimatedUsd": 0.000023299999999999997, "usage": { "prompt_tokens": 61, "completion_tokens": 2, "total_tokens": 63 }, "latencyMs": 571 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000023, "estimatedUsd": 0.000023, "usage": { "prompt_tokens": 60, "completion_tokens": 2, "total_tokens": 62 }, "latencyMs": 601 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.0000286, "estimatedUsd": 0.0000286, "usage": { "prompt_tokens": 62, "completion_tokens": 4, "total_tokens": 66 }, "latencyMs": 435 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0001716, "estimatedUsd": 0.0001716, "usage": { "prompt_tokens": 97, "completion_tokens": 57, "total_tokens": 154 }, "latencyMs": 463 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0001634, "estimatedUsd": 0.0001634, "usage": { "prompt_tokens": 78, "completion_tokens": 56, "total_tokens": 134 }, "latencyMs": 465 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000021999999999999996, "estimatedUsd": 0.000021999999999999996, "usage": { "prompt_tokens": 65, "completion_tokens": 1, "total_tokens": 66 }, "latencyMs": 371 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0001608, "estimatedUsd": 0.0001608, "usage": { "prompt_tokens": 86, "completion_tokens": 54, "total_tokens": 140 }, "latencyMs": 446 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0000844, "estimatedUsd": 0.0000844, "usage": { "prompt_tokens": 73, "completion_tokens": 25, "total_tokens": 98 }, "latencyMs": 456 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0001155, "estimatedUsd": 0.0001155, "usage": { "prompt_tokens": 60, "completion_tokens": 39, "total_tokens": 99 }, "latencyMs": 589 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000020799999999999997, "estimatedUsd": 0.000020799999999999997, "usage": { "prompt_tokens": 61, "completion_tokens": 1, "total_tokens": 62 }, "latencyMs": 340 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000019299999999999998, "estimatedUsd": 0.000019299999999999998, "usage": { "prompt_tokens": 56, "completion_tokens": 1, "total_tokens": 57 }, "latencyMs": 431 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0000221, "estimatedUsd": 0.0000221, "usage": { "prompt_tokens": 57, "completion_tokens": 2, "total_tokens": 59 }, "latencyMs": 343 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0000196, "estimatedUsd": 0.0000196, "usage": { "prompt_tokens": 57, "completion_tokens": 1, "total_tokens": 58 }, "latencyMs": 498 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000184, "estimatedUsd": 0.0000184, "usage": { "prompt_tokens": 53, "completion_tokens": 1, "total_tokens": 54 }, "latencyMs": 432 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0000178, "estimatedUsd": 0.0000178, "usage": { "prompt_tokens": 51, "completion_tokens": 1, "total_tokens": 52 }, "latencyMs": 584 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper extraction tasks succeeded, but arithmetic failed completely.", "spendUsd": 0.0000509, "estimatedUsd": 0.0000509, "usage": { "prompt_tokens": 78, "completion_tokens": 11, "total_tokens": 89 }, "latencyMs": 1662 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0000505, "estimatedUsd": 0.0000505, "usage": { "prompt_tokens": 85, "completion_tokens": 10, "total_tokens": 95 }, "latencyMs": 513 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.0000447, "estimatedUsd": 0.0000447, "usage": { "prompt_tokens": 74, "completion_tokens": 9, "total_tokens": 83 }, "latencyMs": 713 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "missing": [], "constraintFailures": [ "wordCount=7, max=6" ], "wordCount": 7, "sentenceCount": 1 }, "output": "The True Cost of a Correct Answer", "spendUsd": 0.0000373, "estimatedUsd": 0.0000373, "usage": { "prompt_tokens": 66, "completion_tokens": 7, "total_tokens": 73 }, "latencyMs": 555 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00011179999999999998, "estimatedUsd": 0.00011179999999999998, "usage": { "prompt_tokens": 81, "completion_tokens": 35, "total_tokens": 116 }, "latencyMs": 440 }, { "taskId": "code-001", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000027199999999999997, "estimatedUsd": 0.000027199999999999997, "usage": { "prompt_tokens": 74, "completion_tokens": 2, "total_tokens": 76 }, "latencyMs": 619 }, { "taskId": "code-002", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000030399999999999997, "estimatedUsd": 0.000030399999999999997, "usage": { "prompt_tokens": 68, "completion_tokens": 4, "total_tokens": 72 }, "latencyMs": 477 }, { "taskId": "code-003", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000026999999999999996, "estimatedUsd": 0.000026999999999999996, "usage": { "prompt_tokens": 65, "completion_tokens": 3, "total_tokens": 68 }, "latencyMs": 591 }, { "taskId": "code-004", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.00011029999999999998, "estimatedUsd": 0.00011029999999999998, "usage": { "prompt_tokens": 76, "completion_tokens": 35, "total_tokens": 111 }, "latencyMs": 528 }, { "taskId": "code-005", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0000296, "estimatedUsd": 0.0000296, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 512 }, { "taskId": "code-006", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0000202, "estimatedUsd": 0.0000202, "usage": { "prompt_tokens": 59, "completion_tokens": 1, "total_tokens": 60 }, "latencyMs": 1683 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Sonnet", "spendUsd": 0.000018500000000000002, "estimatedUsd": 0.000018500000000000002, "usage": { "prompt_tokens": 45, "completion_tokens": 2, "total_tokens": 47 }, "latencyMs": 501 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0000163, "estimatedUsd": 0.0000163, "usage": { "prompt_tokens": 46, "completion_tokens": 1, "total_tokens": 47 }, "latencyMs": 512 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0000267, "estimatedUsd": 0.0000267, "usage": { "prompt_tokens": 39, "completion_tokens": 6, "total_tokens": 45 }, "latencyMs": 687 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000017, "estimatedUsd": 0.000017, "usage": { "prompt_tokens": 40, "completion_tokens": 2, "total_tokens": 42 }, "latencyMs": 508 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000026799999999999998, "estimatedUsd": 0.000026799999999999998, "usage": { "prompt_tokens": 56, "completion_tokens": 4, "total_tokens": 60 }, "latencyMs": 372 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000016599999999999997, "estimatedUsd": 0.000016599999999999997, "usage": { "prompt_tokens": 47, "completion_tokens": 1, "total_tokens": 48 }, "latencyMs": 649 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000017199999999999998, "estimatedUsd": 0.000017199999999999998, "usage": { "prompt_tokens": 49, "completion_tokens": 1, "total_tokens": 50 }, "latencyMs": 448 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0000367, "estimatedUsd": 0.0000367, "usage": { "prompt_tokens": 64, "completion_tokens": 7, "total_tokens": 71 }, "latencyMs": 1919 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000393, "estimatedUsd": 0.0000393, "usage": { "prompt_tokens": 56, "completion_tokens": 9, "total_tokens": 65 }, "latencyMs": 555 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000017199999999999998, "estimatedUsd": 0.000017199999999999998, "usage": { "prompt_tokens": 49, "completion_tokens": 1, "total_tokens": 50 }, "latencyMs": 780 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0000323, "estimatedUsd": 0.0000323, "usage": { "prompt_tokens": 91, "completion_tokens": 2, "total_tokens": 93 }, "latencyMs": 1157 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0000226, "estimatedUsd": 0.0000226, "usage": { "prompt_tokens": 67, "completion_tokens": 1, "total_tokens": 68 }, "latencyMs": 498 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0000277, "estimatedUsd": 0.0000277, "usage": { "prompt_tokens": 59, "completion_tokens": 4, "total_tokens": 63 }, "latencyMs": 344 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.0001372, "estimatedUsd": 0.0001372, "usage": { "prompt_tokens": 74, "completion_tokens": 46, "total_tokens": 120 }, "latencyMs": 578 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000019, "estimatedUsd": 0.000019, "usage": { "prompt_tokens": 55, "completion_tokens": 1, "total_tokens": 56 }, "latencyMs": 813 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0000231, "estimatedUsd": 0.0000231, "usage": { "prompt_tokens": 52, "completion_tokens": 3, "total_tokens": 55 }, "latencyMs": 419 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0000228, "estimatedUsd": 0.0000228, "usage": { "prompt_tokens": 51, "completion_tokens": 3, "total_tokens": 54 }, "latencyMs": 380 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0000265, "estimatedUsd": 0.0000265, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 655 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00011223, "estimatedUsd": 0.00011223, "usage": { "prompt_tokens": 66, "completion_tokens": 96, "total_tokens": 162 }, "latencyMs": 3934 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00023141999999999998, "estimatedUsd": 0.00023141999999999998, "usage": { "prompt_tokens": 82, "completion_tokens": 225, "total_tokens": 307 }, "latencyMs": 4503 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00009135, "estimatedUsd": 0.00009135, "usage": { "prompt_tokens": 58, "completion_tokens": 76, "total_tokens": 134 }, "latencyMs": 2214 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00011397, "estimatedUsd": 0.00011397, "usage": { "prompt_tokens": 56, "completion_tokens": 103, "total_tokens": 159 }, "latencyMs": 2081 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00014311499999999997, "estimatedUsd": 0.00014311499999999997, "usage": { "prompt_tokens": 69, "completion_tokens": 130, "total_tokens": 199 }, "latencyMs": 9773 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000128325, "estimatedUsd": 0.000128325, "usage": { "prompt_tokens": 63, "completion_tokens": 116, "total_tokens": 179 }, "latencyMs": 3202 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00008612999999999999, "estimatedUsd": 0.00008612999999999999, "usage": { "prompt_tokens": 62, "completion_tokens": 68, "total_tokens": 130 }, "latencyMs": 2566 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000087, "estimatedUsd": 0.000087, "usage": { "prompt_tokens": 62, "completion_tokens": 69, "total_tokens": 131 }, "latencyMs": 7648 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.000225765, "estimatedUsd": 0.000225765, "usage": { "prompt_tokens": 91, "completion_tokens": 214, "total_tokens": 305 }, "latencyMs": 8665 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.000182265, "estimatedUsd": 0.000182265, "usage": { "prompt_tokens": 77, "completion_tokens": 171, "total_tokens": 248 }, "latencyMs": 5511 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00008439, "estimatedUsd": 0.00008439, "usage": { "prompt_tokens": 66, "completion_tokens": 64, "total_tokens": 130 }, "latencyMs": 3524 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": "120", "completion": "38", "latency_ms": "910", "ok": "true" }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": \"120\", \"completion\": \"38\", \"latency_ms\": \"910\", \"ok\": \"true\"}", "spendUsd": 0.000170085, "estimatedUsd": 0.000170085, "usage": { "prompt_tokens": 77, "completion_tokens": 157, "total_tokens": 234 }, "latencyMs": 3899 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00015225, "estimatedUsd": 0.00015225, "usage": { "prompt_tokens": 78, "completion_tokens": 136, "total_tokens": 214 }, "latencyMs": 4333 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.000115275, "estimatedUsd": 0.000115275, "usage": { "prompt_tokens": 65, "completion_tokens": 100, "total_tokens": 165 }, "latencyMs": 4356 }, { "taskId": "classify-001", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00008482499999999999, "estimatedUsd": 0.00008482499999999999, "usage": { "prompt_tokens": 65, "completion_tokens": 65, "total_tokens": 130 }, "latencyMs": 2464 }, { "taskId": "classify-002", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000092655, "estimatedUsd": 0.000092655, "usage": { "prompt_tokens": 63, "completion_tokens": 75, "total_tokens": 138 }, "latencyMs": 2271 }, { "taskId": "classify-003", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000075255, "estimatedUsd": 0.000075255, "usage": { "prompt_tokens": 61, "completion_tokens": 56, "total_tokens": 117 }, "latencyMs": 1866 }, { "taskId": "classify-004", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00007395000000000001, "estimatedUsd": 0.00007395000000000001, "usage": { "prompt_tokens": 60, "completion_tokens": 55, "total_tokens": 115 }, "latencyMs": 3433 }, { "taskId": "classify-005", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00009309, "estimatedUsd": 0.00009309, "usage": { "prompt_tokens": 56, "completion_tokens": 79, "total_tokens": 135 }, "latencyMs": 2179 }, { "taskId": "classify-006", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000024795, "estimatedUsd": 0.000024795, "usage": { "prompt_tokens": 53, "completion_tokens": 2, "total_tokens": 55 }, "latencyMs": 3196 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model aced extraction but failed arithmetic tasks.", "spendUsd": 0.000370185, "estimatedUsd": 0.000370185, "usage": { "prompt_tokens": 81, "completion_tokens": 385, "total_tokens": 466 }, "latencyMs": 9024 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000092655, "estimatedUsd": 0.000092655, "usage": { "prompt_tokens": 87, "completion_tokens": 63, "total_tokens": 150 }, "latencyMs": 3644 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Verbosity increases cost per correct answer.", "spendUsd": 0.000366705, "estimatedUsd": 0.000366705, "usage": { "prompt_tokens": 75, "completion_tokens": 384, "total_tokens": 459 }, "latencyMs": 8093 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost per Correct Answer", "spendUsd": 0.00019183499999999998, "estimatedUsd": 0.00019183499999999998, "usage": { "prompt_tokens": 69, "completion_tokens": 186, "total_tokens": 255 }, "latencyMs": 4608 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00021576, "estimatedUsd": 0.00021576, "usage": { "prompt_tokens": 82, "completion_tokens": 207, "total_tokens": 289 }, "latencyMs": 3684 }, { "taskId": "code-001", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00022837499999999998, "estimatedUsd": 0.00022837499999999998, "usage": { "prompt_tokens": 71, "completion_tokens": 227, "total_tokens": 298 }, "latencyMs": 5345 }, { "taskId": "code-002", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00013137, "estimatedUsd": 0.00013137, "usage": { "prompt_tokens": 74, "completion_tokens": 114, "total_tokens": 188 }, "latencyMs": 2168 }, { "taskId": "code-003", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000180525, "estimatedUsd": 0.000180525, "usage": { "prompt_tokens": 63, "completion_tokens": 176, "total_tokens": 239 }, "latencyMs": 5351 }, { "taskId": "code-004", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00020097, "estimatedUsd": 0.00020097, "usage": { "prompt_tokens": 78, "completion_tokens": 192, "total_tokens": 270 }, "latencyMs": 4530 }, { "taskId": "code-005", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000120495, "estimatedUsd": 0.000120495, "usage": { "prompt_tokens": 59, "completion_tokens": 109, "total_tokens": 168 }, "latencyMs": 2246 }, { "taskId": "code-006", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000099615, "estimatedUsd": 0.000099615, "usage": { "prompt_tokens": 61, "completion_tokens": 84, "total_tokens": 145 }, "latencyMs": 4330 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00008961, "estimatedUsd": 0.00008961, "usage": { "prompt_tokens": 48, "completion_tokens": 79, "total_tokens": 127 }, "latencyMs": 14210 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00006786, "estimatedUsd": 0.00006786, "usage": { "prompt_tokens": 48, "completion_tokens": 54, "total_tokens": 102 }, "latencyMs": 2351 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00005263500000000001, "estimatedUsd": 0.00005263500000000001, "usage": { "prompt_tokens": 41, "completion_tokens": 40, "total_tokens": 81 }, "latencyMs": 12592 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00008613, "estimatedUsd": 0.00008613, "usage": { "prompt_tokens": 42, "completion_tokens": 78, "total_tokens": 120 }, "latencyMs": 3436 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000030885, "estimatedUsd": 0.000030885, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 3491 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000094395, "estimatedUsd": 0.000094395, "usage": { "prompt_tokens": 51, "completion_tokens": 83, "total_tokens": 134 }, "latencyMs": 2522 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000023925, "estimatedUsd": 0.000023925, "usage": { "prompt_tokens": 51, "completion_tokens": 2, "total_tokens": 53 }, "latencyMs": 4565 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00010701, "estimatedUsd": 0.00010701, "usage": { "prompt_tokens": 60, "completion_tokens": 93, "total_tokens": 153 }, "latencyMs": 3946 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00015051000000000002, "estimatedUsd": 0.00015051000000000002, "usage": { "prompt_tokens": 60, "completion_tokens": 143, "total_tokens": 203 }, "latencyMs": 3940 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00007047, "estimatedUsd": 0.00007047, "usage": { "prompt_tokens": 52, "completion_tokens": 55, "total_tokens": 107 }, "latencyMs": 2802 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00004002, "estimatedUsd": 0.00004002, "usage": { "prompt_tokens": 86, "completion_tokens": 3, "total_tokens": 89 }, "latencyMs": 940 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00014094, "estimatedUsd": 0.00014094, "usage": { "prompt_tokens": 66, "completion_tokens": 129, "total_tokens": 195 }, "latencyMs": 2972 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00011266499999999999, "estimatedUsd": 0.00011266499999999999, "usage": { "prompt_tokens": 61, "completion_tokens": 99, "total_tokens": 160 }, "latencyMs": 2426 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.00022010999999999998, "estimatedUsd": 0.00022010999999999998, "usage": { "prompt_tokens": 72, "completion_tokens": 217, "total_tokens": 289 }, "latencyMs": 5805 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000122235, "estimatedUsd": 0.000122235, "usage": { "prompt_tokens": 57, "completion_tokens": 112, "total_tokens": 169 }, "latencyMs": 4342 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000271875, "estimatedUsd": 0.000271875, "usage": { "prompt_tokens": 51, "completion_tokens": 287, "total_tokens": 338 }, "latencyMs": 9041 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00009048000000000001, "estimatedUsd": 0.00009048000000000001, "usage": { "prompt_tokens": 54, "completion_tokens": 77, "total_tokens": 131 }, "latencyMs": 2246 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000241425, "estimatedUsd": 0.000241425, "usage": { "prompt_tokens": 59, "completion_tokens": 248, "total_tokens": 307 }, "latencyMs": 3987 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000009800000000000001, "estimatedUsd": 0.000009800000000000001, "usage": { "prompt_tokens": 66, "completion_tokens": 2, "total_tokens": 68 }, "latencyMs": 2592 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.000058800000000000006, "estimatedUsd": 0.000058800000000000006, "usage": { "prompt_tokens": 82, "completion_tokens": 169, "total_tokens": 251 }, "latencyMs": 4162 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00000896, "estimatedUsd": 0.00000896, "usage": { "prompt_tokens": 58, "completion_tokens": 3, "total_tokens": 61 }, "latencyMs": 1588 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00002828, "estimatedUsd": 0.00002828, "usage": { "prompt_tokens": 56, "completion_tokens": 73, "total_tokens": 129 }, "latencyMs": 1814 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00001022, "estimatedUsd": 0.00001022, "usage": { "prompt_tokens": 69, "completion_tokens": 2, "total_tokens": 71 }, "latencyMs": 1134 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00000938, "estimatedUsd": 0.00000938, "usage": { "prompt_tokens": 63, "completion_tokens": 2, "total_tokens": 65 }, "latencyMs": 6797 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000016800000000000002, "estimatedUsd": 0.000016800000000000002, "usage": { "prompt_tokens": 62, "completion_tokens": 29, "total_tokens": 91 }, "latencyMs": 9612 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000024080000000000003, "estimatedUsd": 0.000024080000000000003, "usage": { "prompt_tokens": 62, "completion_tokens": 55, "total_tokens": 117 }, "latencyMs": 1677 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.0000637, "estimatedUsd": 0.0000637, "usage": { "prompt_tokens": 91, "completion_tokens": 182, "total_tokens": 273 }, "latencyMs": 4722 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.00010626000000000002, "estimatedUsd": 0.00010626000000000002, "usage": { "prompt_tokens": 77, "completion_tokens": 341, "total_tokens": 418 }, "latencyMs": 4879 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000027160000000000004, "estimatedUsd": 0.000027160000000000004, "usage": { "prompt_tokens": 66, "completion_tokens": 64, "total_tokens": 130 }, "latencyMs": 1721 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true}", "spendUsd": 0.00004130000000000001, "estimatedUsd": 0.00004130000000000001, "usage": { "prompt_tokens": 77, "completion_tokens": 109, "total_tokens": 186 }, "latencyMs": 2278 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.000018200000000000002, "estimatedUsd": 0.000018200000000000002, "usage": { "prompt_tokens": 78, "completion_tokens": 26, "total_tokens": 104 }, "latencyMs": 5046 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.00001834, "estimatedUsd": 0.00001834, "usage": { "prompt_tokens": 65, "completion_tokens": 33, "total_tokens": 98 }, "latencyMs": 768 }, { "taskId": "classify-001", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000029820000000000002, "estimatedUsd": 0.000029820000000000002, "usage": { "prompt_tokens": 65, "completion_tokens": 74, "total_tokens": 139 }, "latencyMs": 3856 }, { "taskId": "classify-002", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00000966, "estimatedUsd": 0.00000966, "usage": { "prompt_tokens": 63, "completion_tokens": 3, "total_tokens": 66 }, "latencyMs": 1521 }, { "taskId": "classify-003", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000026180000000000003, "estimatedUsd": 0.000026180000000000003, "usage": { "prompt_tokens": 61, "completion_tokens": 63, "total_tokens": 124 }, "latencyMs": 1829 }, { "taskId": "classify-004", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000024920000000000002, "estimatedUsd": 0.000024920000000000002, "usage": { "prompt_tokens": 60, "completion_tokens": 59, "total_tokens": 119 }, "latencyMs": 8657 }, { "taskId": "classify-005", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000008400000000000001, "estimatedUsd": 0.000008400000000000001, "usage": { "prompt_tokens": 56, "completion_tokens": 2, "total_tokens": 58 }, "latencyMs": 1829 }, { "taskId": "classify-006", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000026180000000000003, "estimatedUsd": 0.000026180000000000003, "usage": { "prompt_tokens": 53, "completion_tokens": 67, "total_tokens": 120 }, "latencyMs": 2237 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model extraction succeeded but failed arithmetic reasoning", "spendUsd": 0.00034818, "estimatedUsd": 0.00034818, "usage": { "prompt_tokens": 81, "completion_tokens": 1203, "total_tokens": 1284 }, "latencyMs": 13055 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000015260000000000003, "estimatedUsd": 0.000015260000000000003, "usage": { "prompt_tokens": 87, "completion_tokens": 11, "total_tokens": 98 }, "latencyMs": 1574 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Verbosity increases cost per correct answer.", "spendUsd": 0.00010682000000000001, "estimatedUsd": 0.00010682000000000001, "usage": { "prompt_tokens": 75, "completion_tokens": 344, "total_tokens": 419 }, "latencyMs": 6162 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost per Correct Answer", "spendUsd": 0.00004144000000000001, "estimatedUsd": 0.00004144000000000001, "usage": { "prompt_tokens": 68, "completion_tokens": 114, "total_tokens": 182 }, "latencyMs": 4088 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00002128, "estimatedUsd": 0.00002128, "usage": { "prompt_tokens": 82, "completion_tokens": 35, "total_tokens": 117 }, "latencyMs": 2431 }, { "taskId": "code-001", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 12" }, "output": "9", "spendUsd": 0.000010500000000000001, "estimatedUsd": 0.000010500000000000001, "usage": { "prompt_tokens": 71, "completion_tokens": 2, "total_tokens": 73 }, "latencyMs": 6964 }, { "taskId": "code-002", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 0.85" }, "output": "", "spendUsd": 0, "estimatedUsd": 0, "usage": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "latencyMs": 90006 }, { "taskId": "code-003", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00000994, "estimatedUsd": 0.00000994, "usage": { "prompt_tokens": 63, "completion_tokens": 4, "total_tokens": 67 }, "latencyMs": 841 }, { "taskId": "code-004", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0000574, "estimatedUsd": 0.0000574, "usage": { "prompt_tokens": 78, "completion_tokens": 166, "total_tokens": 244 }, "latencyMs": 3973 }, { "taskId": "code-005", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000009940000000000001, "estimatedUsd": 0.000009940000000000001, "usage": { "prompt_tokens": 59, "completion_tokens": 6, "total_tokens": 65 }, "latencyMs": 1253 }, { "taskId": "code-006", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000009100000000000001, "estimatedUsd": 0.000009100000000000001, "usage": { "prompt_tokens": 61, "completion_tokens": 2, "total_tokens": 63 }, "latencyMs": 8044 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Grok", "spendUsd": 0.000007560000000000001, "estimatedUsd": 0.000007560000000000001, "usage": { "prompt_tokens": 48, "completion_tokens": 3, "total_tokens": 51 }, "latencyMs": 1071 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000007280000000000001, "estimatedUsd": 0.000007280000000000001, "usage": { "prompt_tokens": 48, "completion_tokens": 2, "total_tokens": 50 }, "latencyMs": 960 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000007420000000000001, "estimatedUsd": 0.000007420000000000001, "usage": { "prompt_tokens": 41, "completion_tokens": 6, "total_tokens": 47 }, "latencyMs": 1236 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000006720000000000001, "estimatedUsd": 0.000006720000000000001, "usage": { "prompt_tokens": 42, "completion_tokens": 3, "total_tokens": 45 }, "latencyMs": 1417 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000009940000000000001, "estimatedUsd": 0.000009940000000000001, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1421 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00002114, "estimatedUsd": 0.00002114, "usage": { "prompt_tokens": 51, "completion_tokens": 50, "total_tokens": 101 }, "latencyMs": 1669 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00003150000000000001, "estimatedUsd": 0.00003150000000000001, "usage": { "prompt_tokens": 51, "completion_tokens": 87, "total_tokens": 138 }, "latencyMs": 3117 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000010080000000000002, "estimatedUsd": 0.000010080000000000002, "usage": { "prompt_tokens": 60, "completion_tokens": 6, "total_tokens": 66 }, "latencyMs": 2019 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000032760000000000005, "estimatedUsd": 0.000032760000000000005, "usage": { "prompt_tokens": 60, "completion_tokens": 87, "total_tokens": 147 }, "latencyMs": 2763 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000022400000000000002, "estimatedUsd": 0.000022400000000000002, "usage": { "prompt_tokens": 52, "completion_tokens": 54, "total_tokens": 106 }, "latencyMs": 3446 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00009716000000000001, "estimatedUsd": 0.00009716000000000001, "usage": { "prompt_tokens": 86, "completion_tokens": 304, "total_tokens": 390 }, "latencyMs": 6151 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000009800000000000001, "estimatedUsd": 0.000009800000000000001, "usage": { "prompt_tokens": 66, "completion_tokens": 2, "total_tokens": 68 }, "latencyMs": 1333 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000009380000000000002, "estimatedUsd": 0.000009380000000000002, "usage": { "prompt_tokens": 61, "completion_tokens": 3, "total_tokens": 64 }, "latencyMs": 935 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.00005376, "estimatedUsd": 0.00005376, "usage": { "prompt_tokens": 72, "completion_tokens": 156, "total_tokens": 228 }, "latencyMs": 37124 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000024500000000000003, "estimatedUsd": 0.000024500000000000003, "usage": { "prompt_tokens": 57, "completion_tokens": 59, "total_tokens": 116 }, "latencyMs": 3287 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00002786, "estimatedUsd": 0.00002786, "usage": { "prompt_tokens": 51, "completion_tokens": 74, "total_tokens": 125 }, "latencyMs": 2575 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00000812, "estimatedUsd": 0.00000812, "usage": { "prompt_tokens": 54, "completion_tokens": 2, "total_tokens": 56 }, "latencyMs": 1065 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4-flash", "label": "DeepSeek V4 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0000399, "estimatedUsd": 0.0000399, "usage": { "prompt_tokens": 59, "completion_tokens": 113, "total_tokens": 172 }, "latencyMs": 3456 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00048780000000000004, "estimatedUsd": 0.00048780000000000004, "usage": { "prompt_tokens": 75, "completion_tokens": 87, "total_tokens": 162 }, "latencyMs": 4147 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00048820000000000005, "estimatedUsd": 0.00048820000000000005, "usage": { "prompt_tokens": 91, "completion_tokens": 82, "total_tokens": 173 }, "latencyMs": 2877 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0001716, "estimatedUsd": 0.0001716, "usage": { "prompt_tokens": 66, "completion_tokens": 18, "total_tokens": 84 }, "latencyMs": 1656 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000352, "estimatedUsd": 0.000352, "usage": { "prompt_tokens": 66, "completion_tokens": 59, "total_tokens": 125 }, "latencyMs": 1824 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00015759999999999998, "estimatedUsd": 0.00015759999999999998, "usage": { "prompt_tokens": 78, "completion_tokens": 11, "total_tokens": 89 }, "latencyMs": 7071 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0001566, "estimatedUsd": 0.0001566, "usage": { "prompt_tokens": 71, "completion_tokens": 13, "total_tokens": 84 }, "latencyMs": 5398 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.0001698, "estimatedUsd": 0.0001698, "usage": { "prompt_tokens": 71, "completion_tokens": 16, "total_tokens": 87 }, "latencyMs": 590 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.0001698, "estimatedUsd": 0.0001698, "usage": { "prompt_tokens": 71, "completion_tokens": 16, "total_tokens": 87 }, "latencyMs": 1241 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0014936, "estimatedUsd": 0.0014936, "usage": { "prompt_tokens": 102, "completion_tokens": 307, "total_tokens": 409 }, "latencyMs": 5121 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0012600000000000003, "estimatedUsd": 0.0012600000000000003, "usage": { "prompt_tokens": 86, "completion_tokens": 259, "total_tokens": 345 }, "latencyMs": 5497 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000328, "estimatedUsd": 0.000328, "usage": { "prompt_tokens": 74, "completion_tokens": 51, "total_tokens": 125 }, "latencyMs": 1557 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0013524000000000001, "estimatedUsd": 0.0013524000000000001, "usage": { "prompt_tokens": 86, "completion_tokens": 280, "total_tokens": 366 }, "latencyMs": 11318 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0006324, "estimatedUsd": 0.0006324, "usage": { "prompt_tokens": 84, "completion_tokens": 117, "total_tokens": 201 }, "latencyMs": 3073 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.001013, "estimatedUsd": 0.001013, "usage": { "prompt_tokens": 73, "completion_tokens": 207, "total_tokens": 280 }, "latencyMs": 4388 }, { "taskId": "classify-001", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0012358000000000002, "estimatedUsd": 0.0012358000000000002, "usage": { "prompt_tokens": 75, "completion_tokens": 257, "total_tokens": 332 }, "latencyMs": 3657 }, { "taskId": "classify-002", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0010748, "estimatedUsd": 0.0010748, "usage": { "prompt_tokens": 70, "completion_tokens": 222, "total_tokens": 292 }, "latencyMs": 3353 }, { "taskId": "classify-003", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0012464000000000002, "estimatedUsd": 0.0012464000000000002, "usage": { "prompt_tokens": 70, "completion_tokens": 261, "total_tokens": 331 }, "latencyMs": 7996 }, { "taskId": "classify-004", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0011012, "estimatedUsd": 0.0011012, "usage": { "prompt_tokens": 70, "completion_tokens": 228, "total_tokens": 298 }, "latencyMs": 5323 }, { "taskId": "classify-005", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00043280000000000006, "estimatedUsd": 0.00043280000000000006, "usage": { "prompt_tokens": 64, "completion_tokens": 78, "total_tokens": 142 }, "latencyMs": 7100 }, { "taskId": "classify-006", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.00009999999999999999, "estimatedUsd": 0.00009999999999999999, "usage": { "prompt_tokens": 62, "completion_tokens": 3, "total_tokens": 65 }, "latencyMs": 501 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded extraction but failed arithmetic tasks.", "spendUsd": 0.0009458000000000001, "estimatedUsd": 0.0009458000000000001, "usage": { "prompt_tokens": 91, "completion_tokens": 186, "total_tokens": 277 }, "latencyMs": 2994 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.001054, "estimatedUsd": 0.001054, "usage": { "prompt_tokens": 96, "completion_tokens": 209, "total_tokens": 305 }, "latencyMs": 3171 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity compounds cost per correct answer.", "spendUsd": 0.0012192, "estimatedUsd": 0.0012192, "usage": { "prompt_tokens": 82, "completion_tokens": 251, "total_tokens": 333 }, "latencyMs": 5776 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The Cost of a Correct Answer", "spendUsd": 0.0017842000000000001, "estimatedUsd": 0.0017842000000000001, "usage": { "prompt_tokens": 77, "completion_tokens": 381, "total_tokens": 458 }, "latencyMs": 1869 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0014284, "estimatedUsd": 0.0014284, "usage": { "prompt_tokens": 90, "completion_tokens": 296, "total_tokens": 386 }, "latencyMs": 2069 }, { "taskId": "code-001", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0004362, "estimatedUsd": 0.0004362, "usage": { "prompt_tokens": 79, "completion_tokens": 74, "total_tokens": 153 }, "latencyMs": 1447 }, { "taskId": "code-002", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00036280000000000004, "estimatedUsd": 0.00036280000000000004, "usage": { "prompt_tokens": 80, "completion_tokens": 57, "total_tokens": 137 }, "latencyMs": 1954 }, { "taskId": "code-003", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.0004968, "estimatedUsd": 0.0004968, "usage": { "prompt_tokens": 72, "completion_tokens": 90, "total_tokens": 162 }, "latencyMs": 2986 }, { "taskId": "code-004", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00048140000000000005, "estimatedUsd": 0.00048140000000000005, "usage": { "prompt_tokens": 83, "completion_tokens": 83, "total_tokens": 166 }, "latencyMs": 2116 }, { "taskId": "code-005", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0009504, "estimatedUsd": 0.0009504, "usage": { "prompt_tokens": 66, "completion_tokens": 195, "total_tokens": 261 }, "latencyMs": 5158 }, { "taskId": "code-006", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0002652, "estimatedUsd": 0.0002652, "usage": { "prompt_tokens": 70, "completion_tokens": 38, "total_tokens": 108 }, "latencyMs": 945 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0003174, "estimatedUsd": 0.0003174, "usage": { "prompt_tokens": 57, "completion_tokens": 54, "total_tokens": 111 }, "latencyMs": 3210 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00037020000000000005, "estimatedUsd": 0.00037020000000000005, "usage": { "prompt_tokens": 57, "completion_tokens": 66, "total_tokens": 123 }, "latencyMs": 1019 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0007036000000000001, "estimatedUsd": 0.0007036000000000001, "usage": { "prompt_tokens": 50, "completion_tokens": 144, "total_tokens": 194 }, "latencyMs": 3689 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00030900000000000003, "estimatedUsd": 0.00030900000000000003, "usage": { "prompt_tokens": 51, "completion_tokens": 54, "total_tokens": 105 }, "latencyMs": 6292 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0010162, "estimatedUsd": 0.0010162, "usage": { "prompt_tokens": 69, "completion_tokens": 209, "total_tokens": 278 }, "latencyMs": 4226 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0001104, "estimatedUsd": 0.0001104, "usage": { "prompt_tokens": 60, "completion_tokens": 6, "total_tokens": 66 }, "latencyMs": 11782 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0005196, "estimatedUsd": 0.0005196, "usage": { "prompt_tokens": 60, "completion_tokens": 99, "total_tokens": 159 }, "latencyMs": 1886 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00047500000000000005, "estimatedUsd": 0.00047500000000000005, "usage": { "prompt_tokens": 69, "completion_tokens": 86, "total_tokens": 155 }, "latencyMs": 6877 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00039139999999999997, "estimatedUsd": 0.00039139999999999997, "usage": { "prompt_tokens": 69, "completion_tokens": 67, "total_tokens": 136 }, "latencyMs": 6284 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000295, "estimatedUsd": 0.000295, "usage": { "prompt_tokens": 63, "completion_tokens": 47, "total_tokens": 110 }, "latencyMs": 1462 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.001167, "estimatedUsd": 0.001167, "usage": { "prompt_tokens": 95, "completion_tokens": 235, "total_tokens": 330 }, "latencyMs": 3154 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0002942, "estimatedUsd": 0.0002942, "usage": { "prompt_tokens": 75, "completion_tokens": 43, "total_tokens": 118 }, "latencyMs": 1552 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0007698000000000001, "estimatedUsd": 0.0007698000000000001, "usage": { "prompt_tokens": 69, "completion_tokens": 153, "total_tokens": 222 }, "latencyMs": 833 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.0005182, "estimatedUsd": 0.0005182, "usage": { "prompt_tokens": 81, "completion_tokens": 92, "total_tokens": 173 }, "latencyMs": 3924 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.0009064, "estimatedUsd": 0.0009064, "usage": { "prompt_tokens": 66, "completion_tokens": 185, "total_tokens": 251 }, "latencyMs": 3172 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0002952, "estimatedUsd": 0.0002952, "usage": { "prompt_tokens": 60, "completion_tokens": 48, "total_tokens": 108 }, "latencyMs": 1942 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0003478, "estimatedUsd": 0.0003478, "usage": { "prompt_tokens": 63, "completion_tokens": 59, "total_tokens": 122 }, "latencyMs": 1313 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.001094, "estimatedUsd": 0.001094, "usage": { "prompt_tokens": 68, "completion_tokens": 227, "total_tokens": 295 }, "latencyMs": 6369 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 171, "detail": "expected 186" }, "output": "171", "spendUsd": 0.000040499999999999995, "estimatedUsd": 0.000040499999999999995, "usage": { "prompt_tokens": 69, "completion_tokens": 4, "total_tokens": 73 }, "latencyMs": 744 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 5, "detail": "expected 8" }, "output": "5", "spendUsd": 0.000049, "estimatedUsd": 0.000049, "usage": { "prompt_tokens": 92, "completion_tokens": 2, "total_tokens": 94 }, "latencyMs": 7114 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00004, "estimatedUsd": 0.00004, "usage": { "prompt_tokens": 62, "completion_tokens": 6, "total_tokens": 68 }, "latencyMs": 1212 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000033, "estimatedUsd": 0.000033, "usage": { "prompt_tokens": 57, "completion_tokens": 3, "total_tokens": 60 }, "latencyMs": 575 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0000405, "estimatedUsd": 0.0000405, "usage": { "prompt_tokens": 72, "completion_tokens": 3, "total_tokens": 75 }, "latencyMs": 391 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000037, "estimatedUsd": 0.000037, "usage": { "prompt_tokens": 65, "completion_tokens": 3, "total_tokens": 68 }, "latencyMs": 327 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000037, "estimatedUsd": 0.000037, "usage": { "prompt_tokens": 65, "completion_tokens": 3, "total_tokens": 68 }, "latencyMs": 4284 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000039999999999999996, "estimatedUsd": 0.000039999999999999996, "usage": { "prompt_tokens": 65, "completion_tokens": 5, "total_tokens": 70 }, "latencyMs": 616 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0001285, "estimatedUsd": 0.0001285, "usage": { "prompt_tokens": 98, "completion_tokens": 53, "total_tokens": 151 }, "latencyMs": 3267 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.00011650000000000001, "estimatedUsd": 0.00011650000000000001, "usage": { "prompt_tokens": 80, "completion_tokens": 51, "total_tokens": 131 }, "latencyMs": 1246 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000037, "estimatedUsd": 0.000037, "usage": { "prompt_tokens": 68, "completion_tokens": 2, "total_tokens": 70 }, "latencyMs": 469 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0001245, "estimatedUsd": 0.0001245, "usage": { "prompt_tokens": 87, "completion_tokens": 54, "total_tokens": 141 }, "latencyMs": 1147 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.000078, "estimatedUsd": 0.000078, "usage": { "prompt_tokens": 78, "completion_tokens": 26, "total_tokens": 104 }, "latencyMs": 2542 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0000885, "estimatedUsd": 0.0000885, "usage": { "prompt_tokens": 66, "completion_tokens": 37, "total_tokens": 103 }, "latencyMs": 1082 }, { "taskId": "classify-001", "category": "classification", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000039, "estimatedUsd": 0.000039, "usage": { "prompt_tokens": 69, "completion_tokens": 3, "total_tokens": 72 }, "latencyMs": 1645 }, { "taskId": "classify-002", "category": "classification", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000035, "estimatedUsd": 0.000035, "usage": { "prompt_tokens": 64, "completion_tokens": 2, "total_tokens": 66 }, "latencyMs": 847 }, { "taskId": "classify-003", "category": "classification", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0000365, "estimatedUsd": 0.0000365, "usage": { "prompt_tokens": 64, "completion_tokens": 3, "total_tokens": 67 }, "latencyMs": 799 }, { "taskId": "classify-004", "category": "classification", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000036, "estimatedUsd": 0.000036, "usage": { "prompt_tokens": 63, "completion_tokens": 3, "total_tokens": 66 }, "latencyMs": 1635 }, { "taskId": "classify-005", "category": "classification", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000315, "estimatedUsd": 0.0000315, "usage": { "prompt_tokens": 57, "completion_tokens": 2, "total_tokens": 59 }, "latencyMs": 1122 }, { "taskId": "classify-006", "category": "classification", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No.", "spendUsd": 0.0000315, "estimatedUsd": 0.0000315, "usage": { "prompt_tokens": 54, "completion_tokens": 3, "total_tokens": 57 }, "latencyMs": 1024 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": false, "grade": { "correct": false, "missing": [], "constraintFailures": [ "wordCount=7, expected=8" ], "wordCount": 7, "sentenceCount": 1 }, "output": "Cheaper extraction succeeded; failed arithmetic reasoning tasks.", "spendUsd": 0.000058, "estimatedUsd": 0.000058, "usage": { "prompt_tokens": 83, "completion_tokens": 11, "total_tokens": 94 }, "latencyMs": 613 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0000635, "estimatedUsd": 0.0000635, "usage": { "prompt_tokens": 91, "completion_tokens": 12, "total_tokens": 103 }, "latencyMs": 632 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.0000535, "estimatedUsd": 0.0000535, "usage": { "prompt_tokens": 77, "completion_tokens": 10, "total_tokens": 87 }, "latencyMs": 845 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost of Each Correct Answer", "spendUsd": 0.0000435, "estimatedUsd": 0.0000435, "usage": { "prompt_tokens": 69, "completion_tokens": 6, "total_tokens": 75 }, "latencyMs": 1359 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 11, "sentenceCount": 1 }, "output": "OpenRouter bases its costs on AI Pricing Guru's direct-provider **list prices**.", "spendUsd": 0.0000705, "estimatedUsd": 0.0000705, "usage": { "prompt_tokens": 84, "completion_tokens": 19, "total_tokens": 103 }, "latencyMs": 825 }, { "taskId": "code-001", "category": "coding", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000041, "estimatedUsd": 0.000041, "usage": { "prompt_tokens": 73, "completion_tokens": 3, "total_tokens": 76 }, "latencyMs": 447 }, { "taskId": "code-002", "category": "coding", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000044999999999999996, "estimatedUsd": 0.000044999999999999996, "usage": { "prompt_tokens": 75, "completion_tokens": 5, "total_tokens": 80 }, "latencyMs": 830 }, { "taskId": "code-003", "category": "coding", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "pattern \\binputPerM\\b" }, "output": "0.5", "spendUsd": 0.000040499999999999995, "estimatedUsd": 0.000040499999999999995, "usage": { "prompt_tokens": 69, "completion_tokens": 4, "total_tokens": 73 }, "latencyMs": 519 }, { "taskId": "code-004", "category": "coding", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.00009499999999999999, "estimatedUsd": 0.00009499999999999999, "usage": { "prompt_tokens": 79, "completion_tokens": 37, "total_tokens": 116 }, "latencyMs": 920 }, { "taskId": "code-005", "category": "coding", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000039, "estimatedUsd": 0.000039, "usage": { "prompt_tokens": 60, "completion_tokens": 6, "total_tokens": 66 }, "latencyMs": 521 }, { "taskId": "code-006", "category": "coding", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000034, "estimatedUsd": 0.000034, "usage": { "prompt_tokens": 62, "completion_tokens": 2, "total_tokens": 64 }, "latencyMs": 375 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Sonnet", "spendUsd": 0.000029, "estimatedUsd": 0.000029, "usage": { "prompt_tokens": 49, "completion_tokens": 3, "total_tokens": 52 }, "latencyMs": 769 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000027499999999999998, "estimatedUsd": 0.000027499999999999998, "usage": { "prompt_tokens": 49, "completion_tokens": 2, "total_tokens": 51 }, "latencyMs": 413 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0000315, "estimatedUsd": 0.0000315, "usage": { "prompt_tokens": 42, "completion_tokens": 7, "total_tokens": 49 }, "latencyMs": 588 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0000265, "estimatedUsd": 0.0000265, "usage": { "prompt_tokens": 44, "completion_tokens": 3, "total_tokens": 47 }, "latencyMs": 849 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0000385, "estimatedUsd": 0.0000385, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 2127 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0000295, "estimatedUsd": 0.0000295, "usage": { "prompt_tokens": 53, "completion_tokens": 2, "total_tokens": 55 }, "latencyMs": 1562 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0000295, "estimatedUsd": 0.0000295, "usage": { "prompt_tokens": 53, "completion_tokens": 2, "total_tokens": 55 }, "latencyMs": 411 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000042999999999999995, "estimatedUsd": 0.000042999999999999995, "usage": { "prompt_tokens": 65, "completion_tokens": 7, "total_tokens": 72 }, "latencyMs": 1960 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00004600000000000001, "estimatedUsd": 0.00004600000000000001, "usage": { "prompt_tokens": 62, "completion_tokens": 10, "total_tokens": 72 }, "latencyMs": 1672 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000030500000000000003, "estimatedUsd": 0.000030500000000000003, "usage": { "prompt_tokens": 55, "completion_tokens": 2, "total_tokens": 57 }, "latencyMs": 753 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00005, "estimatedUsd": 0.00005, "usage": { "prompt_tokens": 91, "completion_tokens": 3, "total_tokens": 94 }, "latencyMs": 651 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000037999999999999995, "estimatedUsd": 0.000037999999999999995, "usage": { "prompt_tokens": 70, "completion_tokens": 2, "total_tokens": 72 }, "latencyMs": 1178 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000039, "estimatedUsd": 0.000039, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 553 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.0001035, "estimatedUsd": 0.0001035, "usage": { "prompt_tokens": 78, "completion_tokens": 43, "total_tokens": 121 }, "latencyMs": 840 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes.", "spendUsd": 0.0000335, "estimatedUsd": 0.0000335, "usage": { "prompt_tokens": 58, "completion_tokens": 3, "total_tokens": 61 }, "latencyMs": 3539 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000032500000000000004, "estimatedUsd": 0.000032500000000000004, "usage": { "prompt_tokens": 53, "completion_tokens": 4, "total_tokens": 57 }, "latencyMs": 2485 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000034, "estimatedUsd": 0.000034, "usage": { "prompt_tokens": 56, "completion_tokens": 4, "total_tokens": 60 }, "latencyMs": 8168 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "mistral-large", "openrouterId": "mistralai/mistral-large-2512", "label": "Mistral Large 3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000037500000000000003, "estimatedUsd": 0.000037500000000000003, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 8099 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.001088, "estimatedUsd": 0.001088, "usage": { "prompt_tokens": 271, "completion_tokens": 91, "total_tokens": 362 }, "latencyMs": 1801 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0017139999999999998, "estimatedUsd": 0.0017139999999999998, "usage": { "prompt_tokens": 287, "completion_tokens": 190, "total_tokens": 477 }, "latencyMs": 2885 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0011020000000000001, "estimatedUsd": 0.0011020000000000001, "usage": { "prompt_tokens": 263, "completion_tokens": 96, "total_tokens": 359 }, "latencyMs": 2959 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.001054, "estimatedUsd": 0.001054, "usage": { "prompt_tokens": 263, "completion_tokens": 88, "total_tokens": 351 }, "latencyMs": 1868 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0012360000000000001, "estimatedUsd": 0.0012360000000000001, "usage": { "prompt_tokens": 276, "completion_tokens": 114, "total_tokens": 390 }, "latencyMs": 2565 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0011120000000000001, "estimatedUsd": 0.0011120000000000001, "usage": { "prompt_tokens": 268, "completion_tokens": 96, "total_tokens": 364 }, "latencyMs": 2025 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.0010379999999999999, "estimatedUsd": 0.0010379999999999999, "usage": { "prompt_tokens": 267, "completion_tokens": 84, "total_tokens": 351 }, "latencyMs": 1887 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.001104, "estimatedUsd": 0.001104, "usage": { "prompt_tokens": 267, "completion_tokens": 95, "total_tokens": 362 }, "latencyMs": 2819 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.0014520000000000002, "estimatedUsd": 0.0014520000000000002, "usage": { "prompt_tokens": 297, "completion_tokens": 143, "total_tokens": 440 }, "latencyMs": 3131 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.001312, "estimatedUsd": 0.001312, "usage": { "prompt_tokens": 281, "completion_tokens": 125, "total_tokens": 406 }, "latencyMs": 2021 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0011899999999999999, "estimatedUsd": 0.0011899999999999999, "usage": { "prompt_tokens": 271, "completion_tokens": 108, "total_tokens": 379 }, "latencyMs": 3002 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.001694, "estimatedUsd": 0.001694, "usage": { "prompt_tokens": 286, "completion_tokens": 187, "total_tokens": 473 }, "latencyMs": 3299 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.001484, "estimatedUsd": 0.001484, "usage": { "prompt_tokens": 283, "completion_tokens": 153, "total_tokens": 436 }, "latencyMs": 2767 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.001084, "estimatedUsd": 0.001084, "usage": { "prompt_tokens": 269, "completion_tokens": 91, "total_tokens": 360 }, "latencyMs": 2973 }, { "taskId": "classify-001", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00087, "estimatedUsd": 0.00087, "usage": { "prompt_tokens": 270, "completion_tokens": 55, "total_tokens": 325 }, "latencyMs": 1479 }, { "taskId": "classify-002", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0013419999999999999, "estimatedUsd": 0.0013419999999999999, "usage": { "prompt_tokens": 266, "completion_tokens": 135, "total_tokens": 401 }, "latencyMs": 3194 }, { "taskId": "classify-003", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000862, "estimatedUsd": 0.000862, "usage": { "prompt_tokens": 266, "completion_tokens": 55, "total_tokens": 321 }, "latencyMs": 1877 }, { "taskId": "classify-004", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0008159999999999999, "estimatedUsd": 0.0008159999999999999, "usage": { "prompt_tokens": 267, "completion_tokens": 47, "total_tokens": 314 }, "latencyMs": 1518 }, { "taskId": "classify-005", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00076, "estimatedUsd": 0.00076, "usage": { "prompt_tokens": 260, "completion_tokens": 40, "total_tokens": 300 }, "latencyMs": 1292 }, { "taskId": "classify-006", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000954, "estimatedUsd": 0.000954, "usage": { "prompt_tokens": 258, "completion_tokens": 73, "total_tokens": 331 }, "latencyMs": 2639 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper extraction failed arithmetic but succeeded parsing tasks.", "spendUsd": 0.002254, "estimatedUsd": 0.002254, "usage": { "prompt_tokens": 287, "completion_tokens": 280, "total_tokens": 567 }, "latencyMs": 4838 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.001214, "estimatedUsd": 0.001214, "usage": { "prompt_tokens": 292, "completion_tokens": 105, "total_tokens": 397 }, "latencyMs": 2730 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "High verbosity compounds cost heavily.", "spendUsd": 0.002266, "estimatedUsd": 0.002266, "usage": { "prompt_tokens": 278, "completion_tokens": 285, "total_tokens": 563 }, "latencyMs": 5505 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost of Correct Answers", "spendUsd": 0.001848, "estimatedUsd": 0.001848, "usage": { "prompt_tokens": 273, "completion_tokens": 217, "total_tokens": 490 }, "latencyMs": 3952 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0014619999999999998, "estimatedUsd": 0.0014619999999999998, "usage": { "prompt_tokens": 287, "completion_tokens": 148, "total_tokens": 435 }, "latencyMs": 2877 }, { "taskId": "code-001", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0013219999999999998, "estimatedUsd": 0.0013219999999999998, "usage": { "prompt_tokens": 280, "completion_tokens": 127, "total_tokens": 407 }, "latencyMs": 2317 }, { "taskId": "code-002", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.001458, "estimatedUsd": 0.001458, "usage": { "prompt_tokens": 279, "completion_tokens": 150, "total_tokens": 429 }, "latencyMs": 2768 }, { "taskId": "code-003", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.001594, "estimatedUsd": 0.001594, "usage": { "prompt_tokens": 269, "completion_tokens": 176, "total_tokens": 445 }, "latencyMs": 3630 }, { "taskId": "code-004", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.001794, "estimatedUsd": 0.001794, "usage": { "prompt_tokens": 282, "completion_tokens": 205, "total_tokens": 487 }, "latencyMs": 2781 }, { "taskId": "code-005", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00125, "estimatedUsd": 0.00125, "usage": { "prompt_tokens": 262, "completion_tokens": 121, "total_tokens": 383 }, "latencyMs": 2925 }, { "taskId": "code-006", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00086, "estimatedUsd": 0.00086, "usage": { "prompt_tokens": 268, "completion_tokens": 54, "total_tokens": 322 }, "latencyMs": 1388 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.001496, "estimatedUsd": 0.001496, "usage": { "prompt_tokens": 253, "completion_tokens": 165, "total_tokens": 418 }, "latencyMs": 2843 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000926, "estimatedUsd": 0.000926, "usage": { "prompt_tokens": 253, "completion_tokens": 70, "total_tokens": 323 }, "latencyMs": 2346 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.001374, "estimatedUsd": 0.001374, "usage": { "prompt_tokens": 246, "completion_tokens": 147, "total_tokens": 393 }, "latencyMs": 2565 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00089, "estimatedUsd": 0.00089, "usage": { "prompt_tokens": 247, "completion_tokens": 66, "total_tokens": 313 }, "latencyMs": 1972 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.001198, "estimatedUsd": 0.001198, "usage": { "prompt_tokens": 266, "completion_tokens": 111, "total_tokens": 377 }, "latencyMs": 2115 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00113, "estimatedUsd": 0.00113, "usage": { "prompt_tokens": 256, "completion_tokens": 103, "total_tokens": 359 }, "latencyMs": 2367 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.001386, "estimatedUsd": 0.001386, "usage": { "prompt_tokens": 255, "completion_tokens": 146, "total_tokens": 401 }, "latencyMs": 2626 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0011319999999999998, "estimatedUsd": 0.0011319999999999998, "usage": { "prompt_tokens": 269, "completion_tokens": 99, "total_tokens": 368 }, "latencyMs": 2050 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00117, "estimatedUsd": 0.00117, "usage": { "prompt_tokens": 264, "completion_tokens": 107, "total_tokens": 371 }, "latencyMs": 2940 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000908, "estimatedUsd": 0.000908, "usage": { "prompt_tokens": 259, "completion_tokens": 65, "total_tokens": 324 }, "latencyMs": 1577 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.002596, "estimatedUsd": 0.002596, "usage": { "prompt_tokens": 296, "completion_tokens": 334, "total_tokens": 630 }, "latencyMs": 4396 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.002054, "estimatedUsd": 0.002054, "usage": { "prompt_tokens": 271, "completion_tokens": 252, "total_tokens": 523 }, "latencyMs": 4764 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.001096, "estimatedUsd": 0.001096, "usage": { "prompt_tokens": 266, "completion_tokens": 94, "total_tokens": 360 }, "latencyMs": 2136 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.0015119999999999999, "estimatedUsd": 0.0015119999999999999, "usage": { "prompt_tokens": 279, "completion_tokens": 159, "total_tokens": 438 }, "latencyMs": 3860 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0011060000000000002, "estimatedUsd": 0.0011060000000000002, "usage": { "prompt_tokens": 262, "completion_tokens": 97, "total_tokens": 359 }, "latencyMs": 2869 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000918, "estimatedUsd": 0.000918, "usage": { "prompt_tokens": 258, "completion_tokens": 67, "total_tokens": 325 }, "latencyMs": 1814 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.001178, "estimatedUsd": 0.001178, "usage": { "prompt_tokens": 259, "completion_tokens": 110, "total_tokens": 369 }, "latencyMs": 2748 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.001026, "estimatedUsd": 0.001026, "usage": { "prompt_tokens": 264, "completion_tokens": 83, "total_tokens": 347 }, "latencyMs": 2336 } ] }