{"version": 1, "date": "2026-11-06", "taskSet": "2026-10-29.1", "tasks": 25, "runsPerTask": 3, "temperature": 0.0, "previousDate": "2026-11-04", "finding": "GPT-6 Luna, GPT-6.1 Terra, GPT-5.4, GPT-6.1 Sol and GPT-5.5 each answered 25 of 25 checkable tasks correctly (100%), so these questions no longer tell them apart.", "categories": ["dates-units", "extraction", "instructions", "knowledge", "reasoning"], "models": [{"model": "gpt-6-luna", "label": "GPT-6 Luna", "where": "OpenAI API", "accuracy": 100.0, "accuracyChange": 0.0, "correct": 25, "errors": 0, "latencyP50Seconds": 1.165, "costPerRunUsd": 0.000506, "costPer100TasksUsd": 0.002024, "tokens": 5062, "consistency": 92.0, "byCategory": {"dates-units": 100.0, "extraction": 100.0, "instructions": 100.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openrouter.ai/api/v1/models read 2026-11-04 (list prices; recheck before publishing)"}, {"model": "gpt-6.1-terra", "label": "GPT-6.1 Terra", "where": "OpenAI API", "accuracy": 100.0, "accuracyChange": 0.0, "correct": 25, "errors": 0, "latencyP50Seconds": 1.191, "costPerRunUsd": 0.003111, "costPer100TasksUsd": 0.012443, "tokens": 3892, "consistency": 88.0, "byCategory": {"dates-units": 100.0, "extraction": 100.0, "instructions": 100.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openrouter.ai/api/v1/models read 2026-11-04 (list prices; recheck before publishing)"}, {"model": "gpt-5.4", "label": "GPT-5.4", "where": "OpenAI API", "accuracy": 100.0, "accuracyChange": null, "correct": 25, "errors": 0, "latencyP50Seconds": 0.967, "costPerRunUsd": 0.004855, "costPer100TasksUsd": 0.01942, "tokens": 3081, "consistency": 96.0, "byCategory": {"dates-units": 100.0, "extraction": 100.0, "instructions": 100.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openrouter.ai/api/v1/models read 2026-11-05"}, {"model": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "where": "OpenAI API", "accuracy": 100.0, "accuracyChange": 0.0, "correct": 25, "errors": 0, "latencyP50Seconds": 1.428, "costPerRunUsd": 0.006815, "costPer100TasksUsd": 0.027259, "tokens": 4070, "consistency": 92.0, "byCategory": {"dates-units": 100.0, "extraction": 100.0, "instructions": 100.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openrouter.ai/api/v1/models read 2026-11-04 (list prices; recheck before publishing)"}, {"model": "gpt-5.5", "label": "GPT-5.5", "where": "OpenAI API", "accuracy": 100.0, "accuracyChange": null, "correct": 25, "errors": 0, "latencyP50Seconds": 1.44, "costPerRunUsd": 0.03646, "costPer100TasksUsd": 0.14584, "tokens": 5756, "consistency": 92.0, "byCategory": {"dates-units": 100.0, "extraction": 100.0, "instructions": 100.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openrouter.ai/api/v1/models read 2026-11-05"}, {"model": "gpt-5.4-mini", "label": "GPT-5.4 mini", "where": "OpenAI API", "accuracy": 96.0, "accuracyChange": null, "correct": 24, "errors": 0, "latencyP50Seconds": 0.732, "costPerRunUsd": 0.001542, "costPer100TasksUsd": 0.006169, "tokens": 3138, "consistency": 96.0, "byCategory": {"dates-units": 100.0, "extraction": 100.0, "instructions": 80.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openrouter.ai/api/v1/models read 2026-11-05"}, {"model": "gpt-4.1", "label": "GPT-4.1", "where": "OpenAI API", "accuracy": 96.0, "accuracyChange": 0.0, "correct": 24, "errors": 0, "latencyP50Seconds": 0.672, "costPerRunUsd": 0.002842, "costPer100TasksUsd": 0.011368, "tokens": 3021, "consistency": 100.0, "byCategory": {"dates-units": 75.0, "extraction": 100.0, "instructions": 100.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openai.com/api/pricing (list prices; recheck before publishing)"}, {"model": "gpt-4o-mini", "label": "GPT-4o mini", "where": "OpenAI API", "accuracy": 92.0, "accuracyChange": 0.0, "correct": 23, "errors": 0, "latencyP50Seconds": 0.687, "costPerRunUsd": 0.000218, "costPer100TasksUsd": 0.000872, "tokens": 3045, "consistency": 96.0, "byCategory": {"dates-units": 75.0, "extraction": 100.0, "instructions": 80.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openai.com/api/pricing (list prices; recheck before publishing)"}, {"model": "gpt-4.1-mini", "label": "GPT-4.1 mini", "where": "OpenAI API", "accuracy": 88.0, "accuracyChange": 0.0, "correct": 22, "errors": 0, "latencyP50Seconds": 0.656, "costPerRunUsd": 0.000575, "costPer100TasksUsd": 0.002299, "tokens": 3033, "consistency": 96.0, "byCategory": {"dates-units": 75.0, "extraction": 100.0, "instructions": 60.0, "knowledge": 100.0, "reasoning": 100.0}, "priceSource": "openai.com/api/pricing (list prices; recheck before publishing)"}], "notReached": [], "missedByAll": [], "sources": {"results": "C:\\Users\\AntiKali\\Documents\\Harness\\ModelBench\\2026-11-06\\results.csv", "methodology": "C:\\Users\\AntiKali\\Documents\\Harness\\ModelBench\\2026-11-06\\methodology.md"}, "harder": {"date": "2026-11-06", "taskSet": "2026-10-29.leap1", "tasks": 7, "runsPerTask": 3, "models": [{"model": "gpt-6.1-terra", "label": "GPT-6.1 Terra", "accuracy": 100.0, "byCategory": {"control": 100.0, "trap": 100.0}, "latencyP50Seconds": 1.473, "costPer100TasksUsd": 0.03897142857142857, "errors": 0}, {"model": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "accuracy": 100.0, "byCategory": {"control": 100.0, "trap": 100.0}, "latencyP50Seconds": 1.682, "costPer100TasksUsd": 0.06341904761904762, "errors": 0}, {"model": "gpt-5.5", "label": "GPT-5.5", "accuracy": 100.0, "byCategory": {"control": 100.0, "trap": 100.0}, "latencyP50Seconds": 3.409, "costPer100TasksUsd": 0.5522142857142858, "errors": 0}, {"model": "gpt-5.4", "label": "GPT-5.4", "accuracy": 100.0, "byCategory": {"control": 100.0, "trap": 100.0}, "latencyP50Seconds": 1.008, "costPer100TasksUsd": 0.08796428571428574, "errors": 0}, {"model": "gpt-6-luna", "label": "GPT-6 Luna", "accuracy": 95.2, "byCategory": {"control": 100.0, "trap": 93.3}, "latencyP50Seconds": 2.13, "costPer100TasksUsd": 0.006601904761904763, "errors": 0}, {"model": "gpt-5.4-mini", "label": "GPT-5.4 mini", "accuracy": 71.4, "byCategory": {"control": 100.0, "trap": 60.0}, "latencyP50Seconds": 0.763, "costPer100TasksUsd": 0.02499571428571429, "errors": 0}, {"model": "gpt-4.1", "label": "GPT-4.1", "accuracy": 66.7, "byCategory": {"control": 100.0, "trap": 53.3}, "latencyP50Seconds": 0.69, "costPer100TasksUsd": 0.047047619047619046, "errors": 0}, {"model": "gpt-4.1-mini", "label": "GPT-4.1 mini", "accuracy": 57.1, "byCategory": {"control": 50.0, "trap": 60.0}, "latencyP50Seconds": 0.711, "costPer100TasksUsd": 0.008754285714285711, "errors": 0}, {"model": "gpt-4o-mini", "label": "GPT-4o mini", "accuracy": 28.6, "byCategory": {"control": 0.0, "trap": 40.0}, "latencyP50Seconds": 0.641, "costPer100TasksUsd": 0.004285238095238096, "errors": 0}], "categories": ["control", "trap"]}}