@nyrra/foundry-ai
Version:
Thin Palantir Foundry provider adapters and model catalog for the Vercel AI SDK.
1,824 lines • 53.5 kB
JSON
{
"date": "2026-09-13",
"scope": "Feature working tree based on 60d10b7; original survey plus focused post-fix xAI request reruns. No model throughput or production deployment claim.",
"sdk": {
"ai": "6.0.224",
"openai": "3.0.84",
"google": "3.0.91"
},
"runs": [
{
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"gitSha": "60d10b7",
"startedAt": "2026-09-13T22:13:21.617Z",
"finishedAt": "2026-09-13T22:18:24.032Z",
"filters": {
"provider": "third-party"
}
},
{
"runId": "2026-09-13T22-11-13.762Z-752f06",
"gitSha": "60d10b7",
"startedAt": "2026-09-13T22:11:14.270Z",
"finishedAt": "2026-09-13T22:11:30.015Z",
"filters": {
"modelId": "gemini-3.7-flash",
"provider": "google"
}
},
{
"runId": "2026-09-13T22-12-25.892Z-bbf34c",
"gitSha": "60d10b7",
"startedAt": "2026-09-13T22:12:26.417Z",
"finishedAt": "2026-09-13T22:12:41.687Z",
"filters": {
"modelId": "gemini-3.8-flash",
"provider": "google"
}
},
{
"runId": "2026-09-13T22-15-25.373Z-a43d1c",
"gitSha": "60d10b7",
"startedAt": "2026-09-13T22:15:25.922Z",
"finishedAt": "2026-09-13T22:15:47.998Z",
"filters": {
"modelId": "grok-4-6",
"provider": "third-party"
}
},
{
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"gitSha": "60d10b7",
"startedAt": "2026-09-13T22:17:31.369Z",
"finishedAt": "2026-09-13T22:18:53.438Z",
"filters": {
"provider": "third-party"
}
},
{
"runId": "2026-09-13T22-19-54.656Z-bf2559",
"gitSha": "60d10b7",
"startedAt": "2026-09-13T22:19:55.193Z",
"finishedAt": "2026-09-13T22:19:56.780Z",
"filters": {
"modelId": "grok-420-non-reasoning-latest",
"provider": "third-party"
}
}
],
"cases": [
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 437,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "messages.generate",
"status": "pass",
"durationMs": 506,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "text.generate",
"status": "pass",
"durationMs": 570,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "text.stream",
"status": "pass",
"durationMs": 768,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 734,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 1078,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 972,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 657,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'clearance_granted' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "vision.image_input",
"status": "proxy-rejected",
"durationMs": 104,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_APICallError",
"message": "Not Found",
"proxyErrorName": "LanguageModelService:LanguageModelNotAvailable"
}
},
{
"provider": "third-party",
"modelId": "qwen3-32b",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 3634,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected false to be true // Object.is equality"
}
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "messages.generate",
"status": "pass",
"durationMs": 4565,
"runId": "2026-09-13T22-17-30.822Z-e8b752"
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "text.stream",
"status": "fail",
"durationMs": 623,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_56d5a719-679f-928c-bb7c-a30c4c844b41:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 1562,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "text.generate",
"status": "pass",
"durationMs": 1855,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 793,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_c0fe2bab-1c53-9409-bc9c-d786bed2721e:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 8309,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 14728,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 7027,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 3778,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-reasoning-latest",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 676,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_fb83622d-8293-9684-987f-d9ed87dbb76d:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "messages.generate",
"status": "pass",
"durationMs": 686,
"runId": "2026-09-13T22-17-30.822Z-e8b752"
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 791,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "text.stream",
"status": "fail",
"durationMs": 650,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "text part msg_d5c57e20-a70e-9e93-96e1-161e98a8a7e0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 1204,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "text part msg_bce17008-8e34-9627-9801-a19b3e251ab2 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 1480,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "text.generate",
"status": "pass",
"durationMs": 6274,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 6163,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 1562,
"runId": "2026-09-13T22-19-54.656Z-bf2559",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 654,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-420-non-reasoning-latest",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 492,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "text part msg_527be33a-0a5c-980a-9b02-52729ebde0c9 not found"
}
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 1018,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "messages.generate",
"status": "pass",
"durationMs": 1199,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "text.generate",
"status": "pass",
"durationMs": 1340,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 1127,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "text.stream",
"status": "pass",
"durationMs": 2132,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 1248,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 7412,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 1094,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'APPROVED' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 961,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3-flash",
"capability": "reasoning.visibility",
"status": "pass",
"durationMs": 4089,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "text.generate",
"status": "pass",
"durationMs": 1076,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "messages.generate",
"status": "pass",
"durationMs": 2074,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 2729,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 2162,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 2285,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "text.stream",
"status": "pass",
"durationMs": 6238,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 9353,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 1743,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'active' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "vision.image_input",
"status": "proxy-rejected",
"durationMs": 108,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_APICallError",
"message": "Not Found",
"proxyErrorName": "LanguageModelService:LanguageModelNotAvailable"
}
},
{
"provider": "third-party",
"modelId": "glm-5-3",
"capability": "reasoning.visibility",
"status": "pass",
"durationMs": 11188,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "messages.generate",
"status": "pass",
"durationMs": 543,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 641,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "text.generate",
"status": "pass",
"durationMs": 713,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "text.stream",
"status": "pass",
"durationMs": 877,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 1163,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 1573,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 1408,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 786,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'Allowed' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "vision.image_input",
"status": "proxy-rejected",
"durationMs": 119,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_APICallError",
"message": "Not Found",
"proxyErrorName": "LanguageModelService:LanguageModelNotAvailable"
}
},
{
"provider": "third-party",
"modelId": "glm-5",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 17528,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected false to be true // Object.is equality"
}
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "messages.generate",
"status": "pass",
"durationMs": 532,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 744,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "text.stream",
"status": "pass",
"durationMs": 643,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 1143,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 1465,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "text.generate",
"status": "pass",
"durationMs": 4026,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 6092,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 1315,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoObjectGeneratedError",
"message": "No object generated: could not parse the response.",
"text": "```json\n{\"status\": \"verified\", \"summary\": \"The oncology topic has been verified.\"}\n```"
}
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 628,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-31b",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 4533,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected false to be true // Object.is equality"
}
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "messages.generate",
"status": "pass",
"durationMs": 338,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 497,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "text.generate",
"status": "pass",
"durationMs": 498,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "text.stream",
"status": "pass",
"durationMs": 453,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 677,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 965,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 893,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 1236,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoObjectGeneratedError",
"message": "No object generated: could not parse the response.",
"text": "```json\n{\n \"status\": \"verified\",\n \"summary\": \"The regulatory status for oncology is verified.\"\n}\n```"
}
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 962,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "gemma-4-26b-a4b",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 4936,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected false to be true // Object.is equality"
}
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "messages.generate",
"status": "pass",
"durationMs": 3875,
"runId": "2026-09-13T22-15-25.373Z-a43d1c"
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "text.stream",
"status": "fail",
"durationMs": 687,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_983b9f9a-6506-90fd-9573-ad26c66f1726:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 1789,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 820,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_86f299c0-8c91-90bb-9b96-f93e4a83c10d:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "text.generate",
"status": "pass",
"durationMs": 2913,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 12985,
"runId": "2026-09-13T22-15-25.373Z-a43d1c",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 18575,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 9064,
"runId": "2026-09-13T22-15-25.373Z-a43d1c",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 5012,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-6",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 820,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_acace1b1-32b8-9ab2-8d27-d4d2cbed44fc:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "messages.generate",
"status": "pass",
"durationMs": 1191,
"runId": "2026-09-13T22-17-30.822Z-e8b752"
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "text.generate",
"status": "pass",
"durationMs": 1121,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 1166,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "text.stream",
"status": "fail",
"durationMs": 1050,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_37d90c9d-ed17-9e21-bd1a-4790fcaf51df:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 622,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_7faf0a1e-9878-94c4-8ab5-224a4e4bf1a0:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 3176,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 10510,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 2686,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 1017,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-5",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 722,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_fce3b2c0-aa99-97ad-b483-21ce78f9ece1:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "messages.generate",
"status": "pass",
"durationMs": 2060,
"runId": "2026-09-13T22-17-30.822Z-e8b752"
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "text.stream",
"status": "fail",
"durationMs": 747,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_056511f5-d6f5-9402-acda-579a3d334cfb:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "text.generate",
"status": "pass",
"durationMs": 1948,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 2419,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 969,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_31aba2ca-826f-9f44-a427-edf5fb5776f8:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 6411,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 5149,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 5535,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 2487,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-4-3",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 4544,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_95659a9d-f2fe-9b4f-af21-75c028dd7d82:0 not found"
}
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "messages.generate",
"status": "pass",
"durationMs": 1355,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 1524,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "text.generate",
"status": "pass",
"durationMs": 1580,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "structured.output.object",
"status": "fail",
"durationMs": 1517,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoObjectGeneratedError",
"message": "No object generated: could not parse the response.",
"text": "**Therapy reduces relapse rates; requires monitoring for liver enzyme elevations.**"
}
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 2879,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "text.stream",
"status": "pass",
"durationMs": 3526,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 2539,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 4115,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "vision.image_input",
"status": "proxy-rejected",
"durationMs": 1217,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_APICallError",
"message": "Bad Request",
"proxyErrorName": "LanguageModelService:LlmHttpClientError"
}
},
{
"provider": "third-party",
"modelId": "nemotron-3-ultra-550b-a55b-nvfp4",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 3263,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected false to be true // Object.is equality"
}
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "messages.generate",
"status": "pass",
"durationMs": 390,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "text.generate",
"status": "pass",
"durationMs": 465,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 631,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "text.stream",
"status": "pass",
"durationMs": 609,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 927,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 1045,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 1245,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 714,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'pending' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "vision.image_input",
"status": "proxy-rejected",
"durationMs": 110,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_APICallError",
"message": "Not Found",
"proxyErrorName": "LanguageModelService:LanguageModelNotAvailable"
}
},
{
"provider": "third-party",
"modelId": "qwen3-235b-a22b-2507",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 12402,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected false to be true // Object.is equality"
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "text.generate",
"status": "fail",
"durationMs": 1,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "messages.generate",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "rid.passthrough",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "ri.language-model-service..language-model.llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "text.stream",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "structured.output.object",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "agent.tool_loop",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 1,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "vision.image_input",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "llama-3-3-nemotron-super-49b-v1-5",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 0,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AI_NoSuchModelError",
"message": "llama-3-3-nemotron-super-49b-v1-5 is enrolled in Foundry, but has no verified provider-compatible proxy route. See docs/third-party-model-support.md."
}
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "messages.generate",
"status": "pass",
"durationMs": 1200,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 1556,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "text.generate",
"status": "pass",
"durationMs": 4923,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "text.stream",
"status": "pass",
"durationMs": 3956,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 1744,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'SIGNAL status: approved.' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 6000,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 11075,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 4226,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'pending' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 3015,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k3",
"capability": "reasoning.visibility",
"status": "pass",
"durationMs": 7749,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "text.generate",
"status": "pass",
"durationMs": 482,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 585,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "messages.generate",
"status": "pass",
"durationMs": 656,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "text.stream",
"status": "pass",
"durationMs": 782,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 790,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 1008,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 7290,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 794,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected 'compliant' to match /verified/i"
}
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 817,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "kimi-k2-5",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 2954,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"name": "AssertionError",
"message": "expected false to be true // Object.is equality"
}
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "messages.generate",
"status": "pass",
"durationMs": 4821,
"runId": "2026-09-13T22-17-30.822Z-e8b752"
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "text.stream",
"status": "fail",
"durationMs": 571,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_9ee92fc1-fad6-98d3-a627-6e9c67d1aa59:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "text.generate",
"status": "pass",
"durationMs": 3154,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "tool.loop.deterministic",
"status": "fail",
"durationMs": 679,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_8cd31b6e-6074-9444-9bf1-cada4f3f0b91:0 not found"
}
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 4157,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 18490,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 24221,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 14900,
"runId": "2026-09-13T22-17-30.822Z-e8b752",
"toolExecutionVerified": true
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 3027,
"runId": "2026-09-13T22-13-21.114Z-debaa3"
},
{
"provider": "third-party",
"modelId": "grok-build-0-1",
"capability": "reasoning.visibility",
"status": "fail",
"durationMs": 502,
"runId": "2026-09-13T22-13-21.114Z-debaa3",
"error": {
"message": "reasoning part rs_cc9ff512-3199-94fe-a402-460dedb26a0c:0 not found"
}
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 1751,
"runId": "2026-09-13T22-11-13.762Z-752f06"
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "text.generate",
"status": "pass",
"durationMs": 2208,
"runId": "2026-09-13T22-11-13.762Z-752f06"
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "messages.generate",
"status": "pass",
"durationMs": 2467,
"runId": "2026-09-13T22-11-13.762Z-752f06"
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "text.stream",
"status": "pass",
"durationMs": 2464,
"runId": "2026-09-13T22-11-13.762Z-752f06"
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 4280,
"runId": "2026-09-13T22-11-13.762Z-752f06"
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 4076,
"runId": "2026-09-13T22-11-13.762Z-752f06"
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 3367,
"runId": "2026-09-13T22-11-13.762Z-752f06",
"toolExecutionVerified": true
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "structured.plus.tools",
"status": "fail",
"durationMs": 5477,
"runId": "2026-09-13T22-11-13.762Z-752f06",
"error": {
"name": "AI_NoOutputGeneratedError",
"message": "No output generated."
}
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 2656,
"runId": "2026-09-13T22-11-13.762Z-752f06"
},
{
"provider": "google",
"modelId": "gemini-3.7-flash",
"capability": "reasoning.visibility",
"status": "skipped",
"durationMs": 0,
"runId": "2026-09-13T22-11-13.762Z-752f06",
"error": {
"name": "CapabilitySkippedError",
"message": "Google reasoning visibility is not currently asserted in this matrix."
}
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "rid.passthrough",
"status": "pass",
"durationMs": 2114,
"runId": "2026-09-13T22-12-25.892Z-bbf34c"
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "messages.generate",
"status": "pass",
"durationMs": 2338,
"runId": "2026-09-13T22-12-25.892Z-bbf34c"
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "text.generate",
"status": "pass",
"durationMs": 2608,
"runId": "2026-09-13T22-12-25.892Z-bbf34c"
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "text.stream",
"status": "pass",
"durationMs": 3389,
"runId": "2026-09-13T22-12-25.892Z-bbf34c"
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "tool.loop.deterministic",
"status": "pass",
"durationMs": 3144,
"runId": "2026-09-13T22-12-25.892Z-bbf34c"
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "structured.output.object",
"status": "pass",
"durationMs": 4108,
"runId": "2026-09-13T22-12-25.892Z-bbf34c"
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "agent.tool_loop",
"status": "pass",
"durationMs": 3484,
"runId": "2026-09-13T22-12-25.892Z-bbf34c",
"toolExecutionVerified": true
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "structured.plus.tools",
"status": "pass",
"durationMs": 4048,
"runId": "2026-09-13T22-12-25.892Z-bbf34c",
"toolExecutionVerified": true
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "vision.image_input",
"status": "pass",
"durationMs": 2202,
"runId": "2026-09-13T22-12-25.892Z-bbf34c"
},
{
"provider": "google",
"modelId": "gemini-3.8-flash",
"capability": "reasoning.visibility",
"status": "skipped",
"durationMs": 2,
"runId": "2026-09-13T22-12-25.892Z-bbf34c",
"error": {
"name": "CapabilitySkippedError",
"message": "Google reasoning visibility is not currently asserted in this matrix."
}
}
]
}