Verified Leaderboard
MCPMark Verified is a stabilized, version-pinned subset of MCPMark's standard tasks for more reliable and reproducible MCP evaluation.
- Single-run evaluation (run-1) only; Pass@4 and Pass^4 are not applicable.
- kimi-k2-7-code filesystem scores come from its second recorded run (published here as run-1); all other services use the first run.
- claude-opus-4-8-max and kimi-k2-6 are scores-only entries—per-task results and trajectories are unavailable.
Updated at 07/20/2026 16:05:27
MCP Server Analysis
Detailed performance analysis by MCP server with single-run Pass@1 metrics
Note: view the JSON below for detailed grade information.
JSON
{
"generated_at": "2026-07-20T16:05:27.844147",
"k": 4,
"overall": {
"kimi-k2-7-code": {
"total_tasks": 127,
"total_agent_execution_time": 51293.48183631897,
"total_input_tokens": 101733361,
"total_output_tokens": 1103489,
"total_tokens": 102836850,
"total_turns": 2131,
"avg_agent_execution_time": 403.8857,
"avg_input_tokens": 801050.0866,
"avg_output_tokens": 8688.8898,
"avg_total_tokens": 809738.9764,
"avg_turns": 16.7795,
"per_run_input_tokens": 101733361,
"per_run_output_tokens": 1103489,
"per_run_cost": null,
"actual_model_name": "kimi-k2.7-code",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.8189,
"std": 0
}
},
"kimi-k3-max": {
"total_tasks": 127,
"total_agent_execution_time": 100822.57696914673,
"total_input_tokens": 97101060,
"total_output_tokens": 1266042,
"total_tokens": 98367102,
"total_turns": 1771,
"avg_agent_execution_time": 793.8786,
"avg_input_tokens": 764575.2756,
"avg_output_tokens": 9968.8346,
"avg_total_tokens": 774544.1102,
"avg_turns": 13.9449,
"per_run_input_tokens": 97101060,
"per_run_output_tokens": 1266042,
"per_run_cost": null,
"actual_model_name": "kimi-k3",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9606,
"std": 0
}
},
"gpt-5-5-xhigh": {
"total_tasks": 127,
"total_agent_execution_time": 73924.10925030708,
"total_input_tokens": 104010723,
"total_output_tokens": 1014141,
"total_tokens": 105024864,
"total_turns": 1935,
"avg_agent_execution_time": 582.0796,
"avg_input_tokens": 818982.0709,
"avg_output_tokens": 7985.3622,
"avg_total_tokens": 826967.4331,
"avg_turns": 15.2362,
"per_run_input_tokens": 104010723,
"per_run_output_tokens": 1014141,
"per_run_cost": null,
"actual_model_name": "gpt-5.5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9291,
"std": 0
}
},
"gpt-5-6-sol-max": {
"total_tasks": 127,
"total_agent_execution_time": 70832.61737942696,
"total_input_tokens": 119625478,
"total_output_tokens": 1198717,
"total_tokens": 120824195,
"total_turns": 1799,
"avg_agent_execution_time": 557.7371,
"avg_input_tokens": 941932.8976,
"avg_output_tokens": 9438.7165,
"avg_total_tokens": 951371.6142,
"avg_turns": 14.1654,
"per_run_input_tokens": 119625478,
"per_run_output_tokens": 1198717,
"per_run_cost": null,
"actual_model_name": "gpt-5.6-sol",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9291,
"std": 0
}
},
"claude-fable-5-max": {
"total_tasks": 127,
"total_agent_execution_time": 39607.626022577286,
"total_input_tokens": 87846158,
"total_output_tokens": 1145145,
"total_tokens": 88991303,
"total_turns": 1508,
"avg_agent_execution_time": 311.8711,
"avg_input_tokens": 691702.0315,
"avg_output_tokens": 9016.8898,
"avg_total_tokens": 700718.9213,
"avg_turns": 11.874,
"per_run_input_tokens": 87846158,
"per_run_output_tokens": 1145145,
"per_run_cost": null,
"actual_model_name": "claude-fable-5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8661,
"std": 0
}
},
"deepseek-v4-pro-max": {
"total_tasks": 127,
"total_agent_execution_time": 18793.9183011055,
"total_input_tokens": 146776305,
"total_output_tokens": 1175426,
"total_tokens": 147951731,
"total_turns": 2247,
"avg_agent_execution_time": 147.9836,
"avg_input_tokens": 1155718.937,
"avg_output_tokens": 9255.3228,
"avg_total_tokens": 1164974.2598,
"avg_turns": 17.6929,
"per_run_input_tokens": 146776305,
"per_run_output_tokens": 1175426,
"per_run_cost": null,
"actual_model_name": "deepseek-v4-pro",
"is_open_source_model": true,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.7165,
"std": 0
}
},
"claude-opus-4-8-max": {
"total_tasks": 127,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "claude-opus-4.8",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.7638,
"std": 0
},
"scores_only": true
},
"kimi-k2-6": {
"total_tasks": 127,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "kimi-k2.6",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.7283,
"std": 0
},
"scores_only": true
}
},
"filesystem": {
"kimi-k2-7-code": {
"total_tasks": 30,
"total_agent_execution_time": 5276.347164154053,
"total_input_tokens": 13389651,
"total_output_tokens": 265777,
"total_tokens": 13655428,
"total_turns": 323,
"avg_agent_execution_time": 175.8782,
"avg_input_tokens": 446321.7,
"avg_output_tokens": 8859.2333,
"avg_total_tokens": 455180.9333,
"avg_turns": 10.7667,
"per_run_input_tokens": 13389651,
"per_run_output_tokens": 265777,
"per_run_cost": null,
"actual_model_name": "kimi-k2.7-code",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.7667,
"std": 0
}
},
"kimi-k3-max": {
"total_tasks": 30,
"total_agent_execution_time": 16042.030032873154,
"total_input_tokens": 10433964,
"total_output_tokens": 279810,
"total_tokens": 10713774,
"total_turns": 274,
"avg_agent_execution_time": 534.7343,
"avg_input_tokens": 347798.8,
"avg_output_tokens": 9327,
"avg_total_tokens": 357125.8,
"avg_turns": 9.1333,
"per_run_input_tokens": 10433964,
"per_run_output_tokens": 279810,
"per_run_cost": null,
"actual_model_name": "kimi-k3",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9333,
"std": 0
}
},
"gpt-5-5-xhigh": {
"total_tasks": 30,
"total_agent_execution_time": 4369.623968839645,
"total_input_tokens": 15761169,
"total_output_tokens": 233551,
"total_tokens": 15994720,
"total_turns": 332,
"avg_agent_execution_time": 145.6541,
"avg_input_tokens": 525372.3,
"avg_output_tokens": 7785.0333,
"avg_total_tokens": 533157.3333,
"avg_turns": 11.0667,
"per_run_input_tokens": 15761169,
"per_run_output_tokens": 233551,
"per_run_cost": null,
"actual_model_name": "gpt-5.5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8667,
"std": 0
}
},
"gpt-5-6-sol-max": {
"total_tasks": 30,
"total_agent_execution_time": 6293.398624181747,
"total_input_tokens": 39057149,
"total_output_tokens": 278963,
"total_tokens": 39336112,
"total_turns": 318,
"avg_agent_execution_time": 209.78,
"avg_input_tokens": 1301904.9667,
"avg_output_tokens": 9298.7667,
"avg_total_tokens": 1311203.7333,
"avg_turns": 10.6,
"per_run_input_tokens": 39057149,
"per_run_output_tokens": 278963,
"per_run_cost": null,
"actual_model_name": "gpt-5.6-sol",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9,
"std": 0
}
},
"claude-fable-5-max": {
"total_tasks": 30,
"total_agent_execution_time": 5172.927497625351,
"total_input_tokens": 7438459,
"total_output_tokens": 291067,
"total_tokens": 7729526,
"total_turns": 240,
"avg_agent_execution_time": 172.4309,
"avg_input_tokens": 247948.6333,
"avg_output_tokens": 9702.2333,
"avg_total_tokens": 257650.8667,
"avg_turns": 8,
"per_run_input_tokens": 7438459,
"per_run_output_tokens": 291067,
"per_run_cost": null,
"actual_model_name": "claude-fable-5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9,
"std": 0
}
},
"deepseek-v4-pro-max": {
"total_tasks": 30,
"total_agent_execution_time": 3594.803786277771,
"total_input_tokens": 15806457,
"total_output_tokens": 319673,
"total_tokens": 16126130,
"total_turns": 310,
"avg_agent_execution_time": 119.8268,
"avg_input_tokens": 526881.9,
"avg_output_tokens": 10655.7667,
"avg_total_tokens": 537537.6667,
"avg_turns": 10.3333,
"per_run_input_tokens": 15806457,
"per_run_output_tokens": 319673,
"per_run_cost": null,
"actual_model_name": "deepseek-v4-pro",
"is_open_source_model": true,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.7667,
"std": 0
}
},
"claude-opus-4-8-max": {
"total_tasks": 30,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "claude-opus-4.8",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8667,
"std": 0
},
"scores_only": true
},
"kimi-k2-6": {
"total_tasks": 30,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "kimi-k2.6",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.7,
"std": 0
},
"scores_only": true
}
},
"github": {
"kimi-k2-7-code": {
"total_tasks": 23,
"total_agent_execution_time": 9516.402571439743,
"total_input_tokens": 16667159,
"total_output_tokens": 191141,
"total_tokens": 16858300,
"total_turns": 303,
"avg_agent_execution_time": 413.7566,
"avg_input_tokens": 724659.087,
"avg_output_tokens": 8310.4783,
"avg_total_tokens": 732969.5652,
"avg_turns": 13.1739,
"per_run_input_tokens": 16667159,
"per_run_output_tokens": 191141,
"per_run_cost": null,
"actual_model_name": "kimi-k2.7-code",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.7391,
"std": 0
}
},
"kimi-k3-max": {
"total_tasks": 23,
"total_agent_execution_time": 16840.33838582039,
"total_input_tokens": 16119310,
"total_output_tokens": 299529,
"total_tokens": 16418839,
"total_turns": 304,
"avg_agent_execution_time": 732.1886,
"avg_input_tokens": 700839.5652,
"avg_output_tokens": 13023,
"avg_total_tokens": 713862.5652,
"avg_turns": 13.2174,
"per_run_input_tokens": 16119310,
"per_run_output_tokens": 299529,
"per_run_cost": null,
"actual_model_name": "kimi-k3",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9565,
"std": 0
}
},
"gpt-5-5-xhigh": {
"total_tasks": 23,
"total_agent_execution_time": 7085.468349695206,
"total_input_tokens": 22347245,
"total_output_tokens": 215803,
"total_tokens": 22563048,
"total_turns": 339,
"avg_agent_execution_time": 308.0638,
"avg_input_tokens": 971619.3478,
"avg_output_tokens": 9382.7391,
"avg_total_tokens": 981002.087,
"avg_turns": 14.7391,
"per_run_input_tokens": 22347245,
"per_run_output_tokens": 215803,
"per_run_cost": null,
"actual_model_name": "gpt-5.5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9565,
"std": 0
}
},
"gpt-5-6-sol-max": {
"total_tasks": 23,
"total_agent_execution_time": 8119.57856297493,
"total_input_tokens": 18172169,
"total_output_tokens": 226687,
"total_tokens": 18398856,
"total_turns": 283,
"avg_agent_execution_time": 353.0252,
"avg_input_tokens": 790094.3043,
"avg_output_tokens": 9855.9565,
"avg_total_tokens": 799950.2609,
"avg_turns": 12.3043,
"per_run_input_tokens": 18172169,
"per_run_output_tokens": 226687,
"per_run_cost": null,
"actual_model_name": "gpt-5.6-sol",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9565,
"std": 0
}
},
"claude-fable-5-max": {
"total_tasks": 23,
"total_agent_execution_time": 5889.735616445541,
"total_input_tokens": 15656185,
"total_output_tokens": 248628,
"total_tokens": 15904813,
"total_turns": 241,
"avg_agent_execution_time": 256.0755,
"avg_input_tokens": 680703.6957,
"avg_output_tokens": 10809.913,
"avg_total_tokens": 691513.6087,
"avg_turns": 10.4783,
"per_run_input_tokens": 15656185,
"per_run_output_tokens": 248628,
"per_run_cost": null,
"actual_model_name": "claude-fable-5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8261,
"std": 0
}
},
"deepseek-v4-pro-max": {
"total_tasks": 23,
"total_agent_execution_time": 3609.3871109485626,
"total_input_tokens": 22618426,
"total_output_tokens": 206734,
"total_tokens": 22825160,
"total_turns": 304,
"avg_agent_execution_time": 156.9299,
"avg_input_tokens": 983409.8261,
"avg_output_tokens": 8988.4348,
"avg_total_tokens": 992398.2609,
"avg_turns": 13.2174,
"per_run_input_tokens": 22618426,
"per_run_output_tokens": 206734,
"per_run_cost": null,
"actual_model_name": "deepseek-v4-pro",
"is_open_source_model": true,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.4783,
"std": 0
}
},
"claude-opus-4-8-max": {
"total_tasks": 23,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "claude-opus-4.8",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.5652,
"std": 0
},
"scores_only": true
},
"kimi-k2-6": {
"total_tasks": 23,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "kimi-k2.6",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.6087,
"std": 0
},
"scores_only": true
}
},
"notion": {
"kimi-k2-7-code": {
"total_tasks": 28,
"total_agent_execution_time": 13420.442340373993,
"total_input_tokens": 26955645,
"total_output_tokens": 354961,
"total_tokens": 27310606,
"total_turns": 555,
"avg_agent_execution_time": 479.3015,
"avg_input_tokens": 962701.6071,
"avg_output_tokens": 12677.1786,
"avg_total_tokens": 975378.7857,
"avg_turns": 19.8214,
"per_run_input_tokens": 26955645,
"per_run_output_tokens": 354961,
"per_run_cost": null,
"actual_model_name": "kimi-k2.7-code",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.8571,
"std": 0
}
},
"kimi-k3-max": {
"total_tasks": 28,
"total_agent_execution_time": 42985.577679395676,
"total_input_tokens": 15129450,
"total_output_tokens": 296944,
"total_tokens": 15426394,
"total_turns": 332,
"avg_agent_execution_time": 1535.1992,
"avg_input_tokens": 540337.5,
"avg_output_tokens": 10605.1429,
"avg_total_tokens": 550942.6429,
"avg_turns": 11.8571,
"per_run_input_tokens": 15129450,
"per_run_output_tokens": 296944,
"per_run_cost": null,
"actual_model_name": "kimi-k3",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9286,
"std": 0
}
},
"gpt-5-5-xhigh": {
"total_tasks": 28,
"total_agent_execution_time": 45423.89243173599,
"total_input_tokens": 15486359,
"total_output_tokens": 209650,
"total_tokens": 15696009,
"total_turns": 409,
"avg_agent_execution_time": 1622.2819,
"avg_input_tokens": 553084.25,
"avg_output_tokens": 7487.5,
"avg_total_tokens": 560571.75,
"avg_turns": 14.6071,
"per_run_input_tokens": 15486359,
"per_run_output_tokens": 209650,
"per_run_cost": null,
"actual_model_name": "gpt-5.5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8929,
"std": 0
}
},
"gpt-5-6-sol-max": {
"total_tasks": 28,
"total_agent_execution_time": 43456.57883286476,
"total_input_tokens": 19367067,
"total_output_tokens": 259772,
"total_tokens": 19626839,
"total_turns": 418,
"avg_agent_execution_time": 1552.0207,
"avg_input_tokens": 691680.9643,
"avg_output_tokens": 9277.5714,
"avg_total_tokens": 700958.5357,
"avg_turns": 14.9286,
"per_run_input_tokens": 19367067,
"per_run_output_tokens": 259772,
"per_run_cost": null,
"actual_model_name": "gpt-5.6-sol",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8571,
"std": 0
}
},
"claude-fable-5-max": {
"total_tasks": 28,
"total_agent_execution_time": 11764.565029621124,
"total_input_tokens": 18740925,
"total_output_tokens": 306168,
"total_tokens": 19047093,
"total_turns": 322,
"avg_agent_execution_time": 420.163,
"avg_input_tokens": 669318.75,
"avg_output_tokens": 10934.5714,
"avg_total_tokens": 680253.3214,
"avg_turns": 11.5,
"per_run_input_tokens": 18740925,
"per_run_output_tokens": 306168,
"per_run_cost": null,
"actual_model_name": "claude-fable-5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8214,
"std": 0
}
},
"deepseek-v4-pro-max": {
"total_tasks": 28,
"total_agent_execution_time": 4277.342442035675,
"total_input_tokens": 20577846,
"total_output_tokens": 287384,
"total_tokens": 20865230,
"total_turns": 498,
"avg_agent_execution_time": 152.7622,
"avg_input_tokens": 734923.0714,
"avg_output_tokens": 10263.7143,
"avg_total_tokens": 745186.7857,
"avg_turns": 17.7857,
"per_run_input_tokens": 20577846,
"per_run_output_tokens": 287384,
"per_run_cost": null,
"actual_model_name": "deepseek-v4-pro",
"is_open_source_model": true,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.7143,
"std": 0
}
},
"claude-opus-4-8-max": {
"total_tasks": 28,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "claude-opus-4.8",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.7857,
"std": 0
},
"scores_only": true
},
"kimi-k2-6": {
"total_tasks": 28,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "kimi-k2.6",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.7321,
"std": 0
},
"scores_only": true
}
},
"playwright": {
"kimi-k2-7-code": {
"total_tasks": 25,
"total_agent_execution_time": 11070.083664894104,
"total_input_tokens": 36879161,
"total_output_tokens": 93241,
"total_tokens": 36972402,
"total_turns": 528,
"avg_agent_execution_time": 442.8033,
"avg_input_tokens": 1475166.44,
"avg_output_tokens": 3729.64,
"avg_total_tokens": 1478896.08,
"avg_turns": 21.12,
"per_run_input_tokens": 36879161,
"per_run_output_tokens": 93241,
"per_run_cost": null,
"actual_model_name": "kimi-k2.7-code",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.84,
"std": 0
}
},
"kimi-k3-max": {
"total_tasks": 25,
"total_agent_execution_time": 13147.619385957718,
"total_input_tokens": 47069042,
"total_output_tokens": 183700,
"total_tokens": 47252742,
"total_turns": 515,
"avg_agent_execution_time": 525.9048,
"avg_input_tokens": 1882761.68,
"avg_output_tokens": 7348,
"avg_total_tokens": 1890109.68,
"avg_turns": 20.6,
"per_run_input_tokens": 47069042,
"per_run_output_tokens": 183700,
"per_run_cost": null,
"actual_model_name": "kimi-k3",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 1,
"std": 0
}
},
"gpt-5-5-xhigh": {
"total_tasks": 25,
"total_agent_execution_time": 8041.704617261887,
"total_input_tokens": 44613781,
"total_output_tokens": 145903,
"total_tokens": 44759684,
"total_turns": 567,
"avg_agent_execution_time": 321.6682,
"avg_input_tokens": 1784551.24,
"avg_output_tokens": 5836.12,
"avg_total_tokens": 1790387.36,
"avg_turns": 22.68,
"per_run_input_tokens": 44613781,
"per_run_output_tokens": 145903,
"per_run_cost": null,
"actual_model_name": "gpt-5.5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.96,
"std": 0
}
},
"gpt-5-6-sol-max": {
"total_tasks": 25,
"total_agent_execution_time": 6074.064154148102,
"total_input_tokens": 38906493,
"total_output_tokens": 83694,
"total_tokens": 38990187,
"total_turns": 536,
"avg_agent_execution_time": 242.9626,
"avg_input_tokens": 1556259.72,
"avg_output_tokens": 3347.76,
"avg_total_tokens": 1559607.48,
"avg_turns": 21.44,
"per_run_input_tokens": 38906493,
"per_run_output_tokens": 83694,
"per_run_cost": null,
"actual_model_name": "gpt-5.6-sol",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 1,
"std": 0
}
},
"claude-fable-5-max": {
"total_tasks": 25,
"total_agent_execution_time": 9676.370019435883,
"total_input_tokens": 40302931,
"total_output_tokens": 133324,
"total_tokens": 40436255,
"total_turns": 431,
"avg_agent_execution_time": 387.0548,
"avg_input_tokens": 1612117.24,
"avg_output_tokens": 5332.96,
"avg_total_tokens": 1617450.2,
"avg_turns": 17.24,
"per_run_input_tokens": 40302931,
"per_run_output_tokens": 133324,
"per_run_cost": null,
"actual_model_name": "claude-fable-5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.92,
"std": 0
}
},
"deepseek-v4-pro-max": {
"total_tasks": 25,
"total_agent_execution_time": 4347.615716218948,
"total_input_tokens": 64567152,
"total_output_tokens": 166880,
"total_tokens": 64734032,
"total_turns": 651,
"avg_agent_execution_time": 173.9046,
"avg_input_tokens": 2582686.08,
"avg_output_tokens": 6675.2,
"avg_total_tokens": 2589361.28,
"avg_turns": 26.04,
"per_run_input_tokens": 64567152,
"per_run_output_tokens": 166880,
"per_run_cost": null,
"actual_model_name": "deepseek-v4-pro",
"is_open_source_model": true,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.72,
"std": 0
}
},
"claude-opus-4-8-max": {
"total_tasks": 25,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "claude-opus-4.8",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.68,
"std": 0
},
"scores_only": true
},
"kimi-k2-6": {
"total_tasks": 25,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "kimi-k2.6",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.76,
"std": 0
},
"scores_only": true
}
},
"postgres": {
"kimi-k2-7-code": {
"total_tasks": 21,
"total_agent_execution_time": 12010.206095457077,
"total_input_tokens": 7841745,
"total_output_tokens": 198369,
"total_tokens": 8040114,
"total_turns": 422,
"avg_agent_execution_time": 571.9146,
"avg_input_tokens": 373416.4286,
"avg_output_tokens": 9446.1429,
"avg_total_tokens": 382862.5714,
"avg_turns": 20.0952,
"per_run_input_tokens": 7841745,
"per_run_output_tokens": 198369,
"per_run_cost": null,
"actual_model_name": "kimi-k2.7-code",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.9048,
"std": 0
}
},
"kimi-k3-max": {
"total_tasks": 21,
"total_agent_execution_time": 11807.011485099792,
"total_input_tokens": 8349294,
"total_output_tokens": 206059,
"total_tokens": 8555353,
"total_turns": 346,
"avg_agent_execution_time": 562.2386,
"avg_input_tokens": 397585.4286,
"avg_output_tokens": 9812.3333,
"avg_total_tokens": 407397.7619,
"avg_turns": 16.4762,
"per_run_input_tokens": 8349294,
"per_run_output_tokens": 206059,
"per_run_cost": null,
"actual_model_name": "kimi-k3",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 1,
"std": 0
}
},
"gpt-5-5-xhigh": {
"total_tasks": 21,
"total_agent_execution_time": 9003.419882774353,
"total_input_tokens": 5802169,
"total_output_tokens": 209234,
"total_tokens": 6011403,
"total_turns": 288,
"avg_agent_execution_time": 428.7343,
"avg_input_tokens": 276293.7619,
"avg_output_tokens": 9963.5238,
"avg_total_tokens": 286257.2857,
"avg_turns": 13.7143,
"per_run_input_tokens": 5802169,
"per_run_output_tokens": 209234,
"per_run_cost": null,
"actual_model_name": "gpt-5.5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 1,
"std": 0
}
},
"gpt-5-6-sol-max": {
"total_tasks": 21,
"total_agent_execution_time": 6888.997205257416,
"total_input_tokens": 4122600,
"total_output_tokens": 349601,
"total_tokens": 4472201,
"total_turns": 244,
"avg_agent_execution_time": 328.0475,
"avg_input_tokens": 196314.2857,
"avg_output_tokens": 16647.6667,
"avg_total_tokens": 212961.9524,
"avg_turns": 11.619,
"per_run_input_tokens": 4122600,
"per_run_output_tokens": 349601,
"per_run_cost": null,
"actual_model_name": "gpt-5.6-sol",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9524,
"std": 0
}
},
"claude-fable-5-max": {
"total_tasks": 21,
"total_agent_execution_time": 7104.027859449387,
"total_input_tokens": 5707658,
"total_output_tokens": 165958,
"total_tokens": 5873616,
"total_turns": 274,
"avg_agent_execution_time": 338.287,
"avg_input_tokens": 271793.2381,
"avg_output_tokens": 7902.7619,
"avg_total_tokens": 279696,
"avg_turns": 13.0476,
"per_run_input_tokens": 5707658,
"per_run_output_tokens": 165958,
"per_run_cost": null,
"actual_model_name": "claude-fable-5",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.8571,
"std": 0
}
},
"deepseek-v4-pro-max": {
"total_tasks": 21,
"total_agent_execution_time": 2964.7692456245422,
"total_input_tokens": 23206424,
"total_output_tokens": 194755,
"total_tokens": 23401179,
"total_turns": 484,
"avg_agent_execution_time": 141.1795,
"avg_input_tokens": 1105067.8095,
"avg_output_tokens": 9274.0476,
"avg_total_tokens": 1114341.8571,
"avg_turns": 23.0476,
"per_run_input_tokens": 23206424,
"per_run_output_tokens": 194755,
"per_run_cost": null,
"actual_model_name": "deepseek-v4-pro",
"is_open_source_model": true,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9048,
"std": 0
}
},
"claude-opus-4-8-max": {
"total_tasks": 21,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "claude-opus-4.8",
"is_open_source_model": false,
"is_reasoning_model": true,
"pass@1": {
"avg": 0.9048,
"std": 0
},
"scores_only": true
},
"kimi-k2-6": {
"total_tasks": 21,
"total_agent_execution_time": null,
"total_input_tokens": null,
"total_output_tokens": null,
"total_tokens": null,
"total_turns": null,
"avg_agent_execution_time": null,
"avg_input_tokens": null,
"avg_output_tokens": null,
"avg_total_tokens": null,
"avg_turns": null,
"per_run_input_tokens": null,
"per_run_output_tokens": null,
"per_run_cost": null,
"actual_model_name": "kimi-k2.6",
"is_open_source_model": true,
"is_reasoning_model": false,
"pass@1": {
"avg": 0.8571,
"std": 0
},
"scores_only": true
}
},
"experiment_name": "verified",
"task_set": "standard",
"task_version": "verified"
}