{
  "id": "cost-per-task-v2",
  "version": 2,
  "created": "2026-07-20",
  "methodology": "Short deterministic tasks mapped to AI API pricing search demand: arithmetic, instruction following, extraction, classification, summarization, and code reasoning. Prompts require compact answers so spend stays bounded and grading can be reproduced without LLM judges.",
  "tasks": [
    {
      "id": "math-001",
      "category": "reasoning",
      "prompt": "Return only the integer answer. A batch job processes 18 files per minute for 7 minutes, then 12 files per minute for 5 minutes. How many files total?",
      "grader": { "type": "number", "value": 186, "tolerance": 0 }
    },
    {
      "id": "math-002",
      "category": "reasoning",
      "prompt": "Return only the integer answer. A model costs $0.30 per 1M input tokens and $2.50 per 1M output tokens. What is the cost in cents for 100,000 input tokens and 20,000 output tokens?",
      "grader": { "type": "number", "value": 8, "tolerance": 0 }
    },
    {
      "id": "math-003",
      "category": "reasoning",
      "prompt": "Return only the integer answer. If a 64,000-token context window is filled to 75%, how many tokens are used?",
      "grader": { "type": "number", "value": 48000, "tolerance": 0 }
    },
    {
      "id": "math-004",
      "category": "reasoning",
      "prompt": "Return only the integer answer. A provider raises output price from $4/M to $6/M. What is the percent increase?",
      "grader": { "type": "number", "value": 50, "tolerance": 0 }
    },
    {
      "id": "math-005",
      "category": "reasoning",
      "prompt": "Return only the integer answer. A cache hit costs 10% of normal input price. Normal input is $2/M. What is the cache-hit price in cents per 1M tokens?",
      "grader": { "type": "number", "value": 20, "tolerance": 0 }
    },
    {
      "id": "math-006",
      "category": "reasoning",
      "prompt": "Return only the integer answer. A crawler finds 9 new models Monday, 14 Tuesday, and removes 5 duplicates. How many unique new models remain?",
      "grader": { "type": "number", "value": 18, "tolerance": 0 }
    },
    {
      "id": "math-007",
      "category": "reasoning",
      "prompt": "Return only the integer answer. A benchmark has 50 tasks. A model gets 37 correct. What is the accuracy percentage rounded to the nearest whole number?",
      "grader": { "type": "number", "value": 74, "tolerance": 0 }
    },
    {
      "id": "math-008",
      "category": "reasoning",
      "prompt": "Return only the decimal answer with no currency symbol. If a run spends $0.42 and gets 14 correct answers, what is dollars per correct answer?",
      "grader": { "type": "number", "value": 0.03, "tolerance": 0.001 }
    },
    {
      "id": "extract-001",
      "category": "extraction",
      "prompt": "Return only JSON with numeric values for prices and context, without currency symbols or units. Text: 'Gemini Flash: input $0.30/M, output $2.50/M, context 1,048,576 tokens.' Extract fields model, inputPerM, outputPerM, context.",
      "grader": { "type": "json_fields", "fields": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 } }
    },
    {
      "id": "extract-002",
      "category": "extraction",
      "prompt": "Return only JSON. Text: 'Z.ai GLM-5.2 charges $1.40 per million input tokens and $4.40 per million output tokens.' Extract provider, model, inputPerM, outputPerM.",
      "grader": { "type": "json_fields", "fields": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 } }
    },
    {
      "id": "extract-003",
      "category": "extraction",
      "prompt": "Return only the single word immediately before 'plan'. Sentence: 'For heavy coding, the Pro plan is cheaper than per-token API use after roughly 35 million output tokens.'",
      "grader": { "type": "exact", "value": "Pro" }
    },
    {
      "id": "extract-004",
      "category": "extraction",
      "prompt": "Return only JSON. Log line: '2026-07-20 model=deepseek-v4-flash prompt=120 completion=38 latency_ms=910 ok=true'. Extract model, prompt, completion, latency_ms, ok.",
      "grader": { "type": "json_fields", "fields": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true } }
    },
    {
      "id": "extract-005",
      "category": "extraction",
      "prompt": "Return only a comma-separated list of model IDs in their original order. Text: 'Tested: gpt-5.6-luna; claude-sonnet-5; gemini-2.5-flash.'",
      "grader": { "type": "regex", "pattern": "^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }
    },
    {
      "id": "extract-006",
      "category": "extraction",
      "prompt": "Return only JSON. Text: 'Anthropic Claude Haiku 4.5: $1 input, $5 output, active.' Extract provider, model, status.",
      "grader": { "type": "json_fields", "fields": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" } }
    },
    {
      "id": "classify-001",
      "category": "classification",
      "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'OpenAI cuts GPT-5.6 Luna output token price by 20%'.",
      "grader": { "type": "exact", "value": "pricing" }
    },
    {
      "id": "classify-002",
      "category": "classification",
      "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'DeepSeek introduces V4 Flash for low-latency chat apps'.",
      "grader": { "type": "exact", "value": "launch" }
    },
    {
      "id": "classify-003",
      "category": "classification",
      "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Claude API errors spike across us-east for 42 minutes'.",
      "grader": { "type": "exact", "value": "outage" }
    },
    {
      "id": "classify-004",
      "category": "classification",
      "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Step-by-step guide to add streaming responses to a chatbot'.",
      "grader": { "type": "exact", "value": "tutorial" }
    },
    {
      "id": "classify-005",
      "category": "classification",
      "prompt": "Return only yes or no. Query: 'cheapest ai api per token'. Is this query primarily cost/comparison intent?",
      "grader": { "type": "exact", "value": "yes" }
    },
    {
      "id": "classify-006",
      "category": "classification",
      "prompt": "Return only yes or no. Query: 'what is a transformer neural network'. Is this query primarily pricing intent?",
      "grader": { "type": "exact", "value": "no" }
    },
    {
      "id": "summ-001",
      "category": "summarization",
      "prompt": "Summarize in exactly 8 words and include these exact words: cheaper, extraction, failed, arithmetic. Text: 'The cheaper model answered all extraction tasks correctly but failed most multi-step arithmetic, making it a good fit for structured parsing rather than reasoning.'",
      "grader": { "type": "contains_all", "values": ["cheaper", "extraction", "failed", "arithmetic"], "wordCount": 8 }
    },
    {
      "id": "summ-002",
      "category": "summarization",
      "prompt": "Return these three search phrases exactly as written, lowercase and separated by commas with no extra text: ai api pricing, cheapest ai api, token calculator. Text: 'Rising searches include ai api pricing, cheapest ai api, and token calculator, all pointing toward cost-conscious developer intent.'",
      "grader": { "type": "regex", "pattern": "^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }
    },
    {
      "id": "summ-003",
      "category": "summarization",
      "prompt": "Return one sentence under 12 words and include the exact words verbosity and cost. Text: 'A model with high accuracy but very high output verbosity can lose on dollars per correct answer because every extra token compounds cost.'",
      "grader": { "type": "contains_all", "values": ["verbosity", "cost"], "maxWords": 11, "sentenceCount": 1 }
    },
    {
      "id": "summ-004",
      "category": "summarization",
      "prompt": "Return only a title under 7 words that includes the exact words cost and correct. Text: 'We compare leading AI models by how much each correct benchmark answer costs, not just accuracy.'",
      "grader": { "type": "contains_all", "values": ["cost", "correct"], "maxWords": 6 }
    },
    {
      "id": "summ-005",
      "category": "summarization",
      "prompt": "Return exactly one sentence that includes OpenRouter and the exact phrase list prices. Text: 'OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.'",
      "grader": { "type": "contains_all", "values": ["OpenRouter", "list prices"], "sentenceCount": 1 }
    },
    {
      "id": "code-001",
      "category": "coding",
      "prompt": "Return only the output. JavaScript: const xs=[3,1,4,1,5]; console.log(xs.filter(x=>x>2).reduce((a,b)=>a+b,0));",
      "grader": { "type": "exact", "value": "12" }
    },
    {
      "id": "code-002",
      "category": "coding",
      "prompt": "Return only the output. Python: prices={'in':0.3,'out':2.5}; print(round(prices['in']*2 + prices['out']*0.1, 2))",
      "grader": { "type": "exact", "value": "0.85" }
    },
    {
      "id": "code-003",
      "category": "coding",
      "prompt": "Return only the missing JavaScript expression. Complete: const cost = (inputTokens * ___ + outputTokens * outputPerM) / 1_000_000;",
      "grader": { "type": "regex", "pattern": "\\binputPerM\\b" }
    },
    {
      "id": "code-004",
      "category": "coding",
      "prompt": "Return only JSON. For rows [{ok:true,cost:0.02},{ok:false,cost:0.03},{ok:true,cost:0.01}], compute correct, totalCost, costPerCorrect.",
      "grader": { "type": "json_fields", "fields": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 } }
    },
    {
      "id": "code-005",
      "category": "coding",
      "prompt": "Return only the output. JavaScript: console.log(['gpt','claude','glm'].map(s=>s.length).join('-'));",
      "grader": { "type": "exact", "value": "3-6-3" }
    },
    {
      "id": "code-006",
      "category": "coding",
      "prompt": "Return only the called function name immediately after await. Snippet: async function call(){ const r = await fetch(url, opts); return r.json(); }",
      "grader": { "type": "exact", "value": "fetch" }
    },
    {
      "id": "logic-001",
      "category": "instruction",
      "prompt": "Return only the third item alphabetically: Luna, Flash, Sonnet, Grok.",
      "grader": { "type": "exact", "value": "Luna" }
    },
    {
      "id": "logic-002",
      "category": "instruction",
      "prompt": "Return only the word that appears twice: token, price, model, token, output.",
      "grader": { "type": "exact", "value": "token" }
    },
    {
      "id": "logic-003",
      "category": "instruction",
      "prompt": "Return only the reversed string: glm-5.2",
      "grader": { "type": "exact", "value": "2.5-mlg" }
    },
    {
      "id": "logic-004",
      "category": "instruction",
      "prompt": "Return only the uppercase acronym from: cost per correct answer.",
      "grader": { "type": "exact", "value": "CPCA" }
    },
    {
      "id": "logic-005",
      "category": "instruction",
      "prompt": "Return only a comma-separated list of the two providers with names starting with G: OpenAI, Google, Anthropic, Groq, Mistral.",
      "grader": { "type": "regex", "pattern": "^Google\\s*,\\s*Groq$" }
    },
    {
      "id": "logic-006",
      "category": "instruction",
      "prompt": "Return only valid or invalid. JSON string: {\"model\":\"gpt\",\"cost\":0.01}",
      "grader": { "type": "exact", "value": "valid" }
    },
    {
      "id": "logic-007",
      "category": "instruction",
      "prompt": "Return only valid or invalid. JSON string: {model:\"gpt\",cost:0.01}",
      "grader": { "type": "exact", "value": "invalid" }
    },
    {
      "id": "logic-008",
      "category": "instruction",
      "prompt": "Return only the smallest price: $1.25/M, $0.30/M, $2.00/M, $0.14/M.",
      "grader": { "type": "exact", "value": "$0.14/M" }
    },
    {
      "id": "logic-009",
      "category": "instruction",
      "prompt": "Return only the final model ID. Chain: start=gpt-5; replace gpt with glm; append .2; prefix z-ai-.",
      "grader": { "type": "exact", "value": "z-ai-glm-5.2" }
    },
    {
      "id": "logic-010",
      "category": "instruction",
      "prompt": "Return only the count of unique providers: OpenAI, Anthropic, OpenAI, Google, Z.ai, Google.",
      "grader": { "type": "number", "value": 4, "tolerance": 0 }
    },
    {
      "id": "pricing-001",
      "category": "pricing",
      "prompt": "Return only cheaper or expensive. Model A costs $0.10/M input and $0.30/M output. Model B costs $0.30/M input and $2.50/M output. For any positive input/output workload, Model A is what relative to Model B?",
      "grader": { "type": "exact", "value": "cheaper" }
    },
    {
      "id": "pricing-002",
      "category": "pricing",
      "prompt": "Return only the model name. A got 10 correct for $0.20. B got 8 correct for $0.08. Which has lower dollars per correct answer?",
      "grader": { "type": "exact", "value": "B" }
    },
    {
      "id": "pricing-003",
      "category": "pricing",
      "prompt": "Return only the integer answer. If output tokens are 4x more expensive than input tokens, how many input-token equivalents is 250 output tokens?",
      "grader": { "type": "number", "value": 1000, "tolerance": 0 }
    },
    {
      "id": "pricing-004",
      "category": "pricing",
      "prompt": "Return only JSON. A run used 1000 prompt tokens and 500 completion tokens. Prices are $1/M input and $6/M output. Compute inputCost, outputCost, totalCost in dollars.",
      "grader": { "type": "json_fields", "fields": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 } }
    },
    {
      "id": "pricing-005",
      "category": "pricing",
      "prompt": "Return only yes or no. If a model is free for input but charges for output, can a verbose wrong answer still cost money?",
      "grader": { "type": "exact", "value": "yes" }
    },
    {
      "id": "pricing-006",
      "category": "pricing",
      "prompt": "Return only the ratio as N:1. Output costs $15/M and input costs $3/M.",
      "grader": { "type": "exact", "value": "5:1" }
    },
    {
      "id": "pricing-007",
      "category": "pricing",
      "prompt": "Return only the integer answer. A $5 pilot budget has already spent $1.75. How many cents remain?",
      "grader": { "type": "number", "value": 325, "tolerance": 0 }
    },
    {
      "id": "pricing-008",
      "category": "pricing",
      "prompt": "Return only the best metric name: cost per prompt, cost per correct answer, or total tokens. We need to compare accuracy and spend together.",
      "grader": { "type": "exact", "value": "cost per correct answer" }
    }
  ]
}