{ "buckets": [ 16000, 32000, 64000, 128000, 256000, 512000 ], "capacityClasses": [ { "class": "capacity_out", "count": 15, "description": { "en": "The model's canonical context window cannot hold this bucket.", "tr": "Modelin kanonik bağlam penceresi bu kovaya yetmiyor." } }, { "class": "native_capacity_out", "count": 3, "description": { "en": "The request exceeds the window under the model's own tokenization.", "tr": "İstek, modelin kendi tokenizasyonunda pencereyi aşıyor." } }, { "class": "provider_capacity_reject", "count": 8, "description": { "en": "The provider rejected the request despite its advertised window.", "tr": "Sağlayıcı, ilan edilen pencereye rağmen isteği reddetti." } } ], "category": { "en": "Long context", "tr": "Uzun bağlam" }, "description": { "en": "The same five questions repeat across six context lengths; question, answer and evidence stay fixed while only the context grows.", "tr": "Aynı beş soru altı bağlam uzunluğunda tekrarlanır; soru, cevap ve kanıt sabit kalır, yalnız bağlam uzar." }, "generatedAt": "2026-08-07T09:00:00Z", "id": "rewo-contextcurve-tr-v1", "itemCount": 5, "judge": { "consistencyRate": 0.8810572687224669, "model": "deepseek/deepseek-v4-pro", "packets": 454, "qualification": "exploratory_unqualified" }, "kind": "context_curve", "models": [ { "costUsd": "46.44200375", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } } ], "developerId": "openai", "exclusion": null, "formatDeviations": 1, "formatFailures": 0, "modelId": "openai/gpt-5.6-sol", "name": "GPT-5.6 Sol", "provider": "OpenAI", "ranked": true, "score": { "en": 94.16666666666666, "overall": 97.06666666666668, "tr": 99.0 } }, { "costUsd": "17.806138", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9199999999999999, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.8666666666666667, "expected": 3, "status": "measured" } } ], "developerId": "google", "exclusion": null, "formatDeviations": 0, "formatFailures": 1, "modelId": "google/gemini-3.1-pro-preview", "name": "Gemini 3.1 Pro", "provider": "Google", "ranked": true, "score": { "en": 100.0, "overall": 83.0, "tr": 71.66666666666666 } }, { "costUsd": "0.834683360", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8800000000000001, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.7999999999999999, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.9818181818181818, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9696969696969697, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.8151515151515151, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.8030303030303031, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.75, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.7, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.6133333333333333, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.4666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.8333333333333334, "expected": 3, "status": "measured" } } ], "developerId": "deepseek", "exclusion": null, "formatDeviations": 0, "formatFailures": 1, "modelId": "deepseek/deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash 0731", "provider": "SiliconFlow", "ranked": true, "score": { "en": 97.08333333333333, "overall": 82.78484848484848, "tr": 73.25252525252526 } }, { "costUsd": "7.4637680", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.9818181818181818, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9696969696969697, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.9, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.8333333333333334, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 2, "status": "capacity_out" }, "overall": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 5, "status": "capacity_out" }, "tr": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 3, "status": "capacity_out" } } ], "developerId": "xai", "exclusion": null, "formatDeviations": 0, "formatFailures": 0, "modelId": "x-ai/grok-4.5", "name": "Grok 4.5", "provider": "xAI", "ranked": true, "score": { "en": 83.33333333333334, "overall": 82.74242424242425, "tr": 82.34848484848486 } }, { "costUsd": "9.29244775", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.8933333333333333, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.6666666666666666, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.4666666666666666, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.3333333333333333, "expected": 3, "status": "measured" } } ], "developerId": "openai", "exclusion": null, "formatDeviations": 2, "formatFailures": 0, "modelId": "openai/gpt-5.6-terra", "name": "GPT-5.6 Terra", "provider": "OpenAI", "ranked": true, "score": { "en": 80.0, "overall": 77.33333333333333, "tr": 75.55555555555556 } }, { "costUsd": "0.933322375", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9714285714285713, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9523809523809524, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.7933333333333333, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.7666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.9333333333333332, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.6799999999999999, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.4666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8171428571428571, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6952380952380951, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.9333333333333333, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.8888888888888888, "expected": 3, "status": "measured" } } ], "developerId": "openai", "exclusion": null, "formatDeviations": 1, "formatFailures": 0, "modelId": "openai/gpt-5.6-luna", "name": "GPT-5.6 Luna", "provider": "OpenAI", "ranked": true, "score": { "en": 92.49999999999999, "overall": 76.97619047619047, "tr": 66.62698412698413 } }, { "costUsd": "9.4162806", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9714285714285715, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9523809523809524, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9714285714285715, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9523809523809524, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9714285714285715, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9523809523809524, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 2, "status": "provider_capacity_reject" }, "overall": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 5, "status": "provider_capacity_reject" }, "tr": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 3, "status": "provider_capacity_reject" } } ], "developerId": "moonshot", "exclusion": null, "formatDeviations": 0, "formatFailures": 0, "modelId": "moonshotai/kimi-k3", "name": "Kimi K3", "provider": "Wafer", "ranked": true, "score": { "en": 83.33333333333334, "overall": 75.9047619047619, "tr": 70.95238095238095 } }, { "costUsd": "6.5618410528", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9418181818181818, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.903030303030303, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.6, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.3333333333333333, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8800000000000001, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.7999999999999999, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8151515151515151, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6919191919191919, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.6333333333333333, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.5, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.6904761904761905, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.5952380952380952, "expected": 3, "status": "measured" } } ], "developerId": "deepseek", "exclusion": null, "formatDeviations": 2, "formatFailures": 0, "modelId": "deepseek/deepseek-v4-pro", "name": "DeepSeek V4 Pro", "provider": "Novita", "ranked": true, "score": { "en": 98.33333333333333, "overall": 74.13722943722944, "tr": 58.00649350649351 } }, { "costUsd": "4.58834296", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.8571428571428571, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.7619047619047619, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.8171428571428571, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.6952380952380953, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 3, "status": "provider_capacity_reject" } } ], "developerId": "zai", "exclusion": null, "formatDeviations": 2, "formatFailures": 0, "modelId": "z-ai/glm-5.2", "name": "GLM 5.2", "provider": "BaseTen", "ranked": true, "score": { "en": 100.0, "overall": 71.63809523809523, "tr": 52.730158730158735 } }, { "costUsd": "1.0640094", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.9714285714285713, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9523809523809524, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 1.0, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 3, "status": "native_capacity_out" } }, { "bucket": 512000, "en": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 2, "status": "capacity_out" }, "overall": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 5, "status": "capacity_out" }, "tr": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 3, "status": "capacity_out" } } ], "developerId": "alibaba-qwen", "exclusion": null, "formatDeviations": 0, "formatFailures": 0, "modelId": "qwen/qwen3.6-27b", "name": "Qwen3.6 27B", "provider": "CoreWeave", "ranked": true, "score": { "en": 83.33333333333334, "overall": 67.32380952380953, "tr": 56.65079365079366 } }, { "costUsd": "10.685222", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.8800000000000001, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.7999999999999999, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.6799999999999999, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.4666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.5, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.8, "dispatched": 5, "evidenceF1": 0.5818181818181818, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6363636363636364, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 0.6666666666666666, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.5999999999999999, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.5555555555555555, "expected": 3, "status": "measured" } } ], "developerId": "alibaba-qwen", "exclusion": null, "formatDeviations": 1, "formatFailures": 0, "modelId": "qwen/qwen3.8-max", "name": "Qwen3.8 Max", "provider": "Alibaba", "ranked": true, "score": { "en": 90.0, "overall": 66.04242424242423, "tr": 50.07070707070707 } }, { "costUsd": "38.748280", "curve": [ { "bucket": 16000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.9714285714285713, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9523809523809524, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.95, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9166666666666666, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.95, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.9166666666666666, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } } ], "developerId": "anthropic", "exclusion": null, "formatDeviations": 14, "formatFailures": 2, "modelId": "anthropic/claude-opus-5", "name": "Claude Opus 5", "provider": "Anthropic", "ranked": true, "score": { "en": 30.0, "overall": 50.69047619047619, "tr": 64.48412698412699 } }, { "costUsd": "7.70772180", "curve": [ { "bucket": 16000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.2, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 1.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 1.0, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.96, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.9333333333333332, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.9333333333333333, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.8888888888888888, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.9151515151515153, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.8585858585858586, "expected": 3, "status": "measured" } } ], "developerId": "google", "exclusion": null, "formatDeviations": 17, "formatFailures": 0, "modelId": "google/gemini-3.6-flash", "name": "Gemini 3.6 Flash", "provider": "Google", "ranked": true, "score": { "en": 47.5, "overall": 48.842424242424244, "tr": 49.73737373737373 } }, { "costUsd": "15.457162", "curve": [ { "bucket": 16000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.8, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.6666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 0.5, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.48, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.4666666666666666, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.18181818181818182, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.30303030303030304, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 0.5, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.2, "dispatched": 5, "evidenceF1": 0.2, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.0, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 0.5, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.4, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.3333333333333333, "dispatched": 3, "evidenceF1": 0.3333333333333333, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 1.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.4, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.0, "expected": 3, "status": "measured" } } ], "developerId": "anthropic", "exclusion": null, "formatDeviations": 1, "formatFailures": 14, "modelId": "anthropic/claude-sonnet-5", "name": "Claude Sonnet 5", "provider": "Anthropic", "ranked": true, "score": { "en": 58.333333333333336, "overall": 35.64242424242424, "tr": 20.515151515151516 } }, { "costUsd": "1.333185840", "curve": [ { "bucket": 16000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.0, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 0.3333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.6, "dispatched": 5, "evidenceF1": 0.5151515151515151, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.6363636363636364, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.4, "dispatched": 5, "evidenceF1": 0.2, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.6666666666666666, "dispatched": 3, "evidenceF1": 0.3333333333333333, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 0.5, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.2, "dispatched": 5, "evidenceF1": 0.27999999999999997, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.13333333333333333, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 0.5, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.2, "dispatched": 5, "evidenceF1": 0.2, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.0, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.0, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.0, "expected": 3, "status": "measured" } } ], "developerId": "minimax", "exclusion": { "rate": 0.7333333333333333, "reason": "format_failure_rate" }, "formatDeviations": 0, "formatFailures": 22, "modelId": "minimax/minimax-m3", "name": "MiniMax M3", "provider": "GMICloud", "ranked": false, "score": { "en": 24.166666666666664, "overall": 22.30909090909091, "tr": 21.07070707070707 } }, { "costUsd": "0.36108477", "curve": [ { "bucket": 16000, "en": { "accuracy": 0.5, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.2, "dispatched": 5, "evidenceF1": 0.8171428571428571, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.6952380952380953, "expected": 3, "status": "measured" } }, { "bucket": 32000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 1.0, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.7, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.5, "expected": 3, "status": "measured" } }, { "bucket": 64000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.5, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.5, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.5, "expected": 3, "status": "measured" } }, { "bucket": 128000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.8333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.6933333333333334, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.6, "expected": 3, "status": "measured" } }, { "bucket": 256000, "en": { "accuracy": 0.0, "dispatched": 2, "evidenceF1": 0.3333333333333333, "expected": 2, "status": "measured" }, "overall": { "accuracy": 0.0, "dispatched": 5, "evidenceF1": 0.3333333333333333, "expected": 5, "status": "measured" }, "tr": { "accuracy": 0.0, "dispatched": 3, "evidenceF1": 0.3333333333333333, "expected": 3, "status": "measured" } }, { "bucket": 512000, "en": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 2, "status": "capacity_out" }, "overall": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 5, "status": "capacity_out" }, "tr": { "accuracy": null, "dispatched": 0, "evidenceF1": null, "expected": 3, "status": "capacity_out" } } ], "developerId": "google", "exclusion": null, "formatDeviations": 16, "formatFailures": 1, "modelId": "google/gemma-4-31b-it", "name": "Gemma 4 31B", "provider": "Parasail", "ranked": true, "score": { "en": 24.166666666666664, "overall": 17.55238095238095, "tr": 13.142857142857142 } } ], "protocolVersion": "neutral-citations-v3", "ranking": { "excluded": [ { "modelId": "minimax/minimax-m3", "name": "MiniMax M3", "rate": 0.7333333333333333, "rawScore": 22.30909090909091, "reason": "format_failure_rate" } ], "formatFailureRateMax": 0.5 }, "responseCount": 480, "runCompletedAt": "2026-08-06T18:00:00Z", "runId": "full-models-pinned-v1", "schemaVersion": 3, "scoring": { "accuracyWeight": 0.7, "evidenceWeight": 0.3, "uncoveredPolicy": "zero_cell" }, "slices": [ { "caveat": null, "id": "overall", "label": { "en": "Overall", "tr": "Genel" }, "questionsPerBucket": 5 }, { "caveat": null, "id": "tr", "label": { "en": "Turkish", "tr": "Türkçe" }, "questionsPerBucket": 3 }, { "caveat": { "en": "Two questions per bucket: a cell can only be 0, 50 or 100.", "tr": "Kova başına 2 soru: hücre yalnız 0, 50 veya 100 olabilir." }, "id": "en", "label": { "en": "English", "tr": "İngilizce" }, "questionsPerBucket": 2 } ], "status": "published", "subtitle": { "en": "16K–512K context quality curve", "tr": "16K–512K bağlam kalite eğrisi" }, "suiteCommitment": { "leafCount": 2, "merkleRoot": "ca0d86b4a1c0f8d7bdad5efb26e09afee02c92632fd96369e62534735cbb8844", "suiteVersion": "qa-v1/execution-v3" }, "taskKinds": [ { "accuracy": 0.6333333333333333, "kind": "aggregation", "n": 90 }, { "accuracy": 0.7282608695652174, "kind": "multi_hop", "n": 184 }, { "accuracy": 0.7555555555555555, "kind": "single_hop", "n": 90 }, { "accuracy": 0.3111111111111111, "kind": "temporal_coreference", "n": 90 } ], "title": { "en": "ReWo ContextCurve Turkish v1", "tr": "ReWo ContextCurve Türkçe v1" }, "totals": { "candidateCostUsd": "178.7", "cells": 480, "formatFailures": 41, "validResponses": 413 } }