{ "schemaVersion": 1, "snapshotDate": "2026-09-16", "scope": "Requested 20 product categories and all 17 public portfolio entries; assignments are execution plans, not admitted jobs.", "runtime": { "cluster": "bf-benchmarks", "project": "blobfish-ai-429200", "zone": "us-central1-a", "namespace": "bf-benchmarks", "systemMachine": "e2-standard-2", "taskMachine": "e2-standard-4", "maxTaskNodes": 2, "minTaskNodes": 0, "isolation": "gVisor task pods; a bounded dedicated Job for each admitted evaluation", "harnesses": [ "Harbor 0.21.0", "HUD 0.6.17" ], "taskCpuMax": 2, "taskMemoryMiBMax": 4096, "taskTimeoutSecMax": 900, "jobTimeoutSecMax": 10980, "cleanup": "Task pods are removed after collection; finished run Jobs expire after 600 seconds. Retain benchmark outputs, account caches and all world data." }, "profiles": [ { "id": "harbor", "name": "Harbor / CPU Linux", "nextAction": "Freeze the complete release, build task and verifier images, and qualify original positive and negative controls.", "qualifiedGeneralPurpose": true }, { "id": "browser", "name": "Browser / Harbor", "nextAction": "Prepare the complete task roster and benchmark-specific grading; live sites and self-hosted resettable sites require different environments.", "qualifiedGeneralPurpose": false }, { "id": "world", "name": "Multi-service world / Harbor", "nextAction": "Qualify each world service, MCP endpoint, evidence mount and deterministic reset with its original grader.", "qualifiedGeneralPurpose": false }, { "id": "desktop", "name": "Desktop or emulator", "nextAction": "A qualified VM/emulator or grounding adapter, benchmark assets and native evaluator are required; the CPU container lane is insufficient.", "qualifiedGeneralPurpose": false }, { "id": "simulation", "name": "Tool and conversation simulation", "nextAction": "Pin the simulator, policies, domain state and native evaluator; supply required service access and reset behavior.", "qualifiedGeneralPurpose": false }, { "id": "retrieval", "name": "Retrieval and structured evaluation", "nextAction": "Prepare the corpus and relevance labels, embedding/search or answer adapter, and the benchmark-native scoring pipeline.", "qualifiedGeneralPurpose": false }, { "id": "data", "name": "Data and artifact evaluation", "nextAction": "Prepare databases/workbooks and original grading tools; external warehouses need their benchmark-specific access.", "qualifiedGeneralPurpose": false }, { "id": "document", "name": "Document evaluation", "nextAction": "Prepare source documents, parser interface and pinned text/table/layout evaluator images.", "qualifiedGeneralPurpose": false }, { "id": "audio", "name": "Speech input/output", "nextAction": "Qualify an ASR or TTS endpoint, source audio and native scoring; reference-speaker and language conditions must be preserved.", "qualifiedGeneralPurpose": false }, { "id": "voice", "name": "Streaming voice simulation", "nextAction": "Qualify streaming audio, interruption timing, tool/user simulation and separate task, ASR and TTS scores.", "qualifiedGeneralPurpose": false }, { "id": "generation", "name": "Media generation and scoring", "nextAction": "Qualify image/video generation and benchmark-native evaluator images; GPU requirements need a separately bounded execution profile.", "qualifiedGeneralPurpose": false }, { "id": "multimodal", "name": "Image and video understanding", "nextAction": "Prepare licensed media, modality-capable model input and benchmark-native scoring with the original sampling protocol.", "qualifiedGeneralPurpose": false } ], "categories": [ { "id": "browser", "name": "Browser agents", "objective": "Successful completion of natural-language browser tasks.", "profile": "browser", "profileName": "Browser / Harbor", "nextAction": "Prepare the complete task roster and benchmark-specific grading; live sites and self-hosted resettable sites require different environments.", "benchmarks": [ { "id": "webvoyager", "name": "WebVoyager", "href": "/leaderboard/webvoyager", "sourceUrl": "https://github.com/MinorJerry/WebVoyager", "requirements": "Browser agent with live-web access and recorded trajectories.", "companies": [ { "name": "Browser Use", "url": "https://browser-use.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Browserbase / Stagehand", "url": "https://www.browserbase.com/blog/introducing-the-stagehand-api", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Firecrawl", "url": "https://www.firecrawl.dev/", "fit": "Composite system" }, { "name": "Microsoft Playwright", "url": "https://playwright.dev/", "fit": "Composite system" }, { "name": "Skyvern", "url": "https://www.skyvern.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Steel", "url": "https://steel.dev/", "fit": "Composite system" }, { "name": "TinyFish", "url": "https://docs.tinyfish.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "pilot-published", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "browser" }, { "id": "online-mind2web", "name": "Online-Mind2Web", "href": "/leaderboard/online-mind2web", "sourceUrl": "https://github.com/OSU-NLP-Group/Online-Mind2Web", "requirements": "Live browser, action/screenshot trace and dated task snapshot.", "companies": [ { "name": "Browser Use", "url": "https://browser-use.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Browserbase / Stagehand", "url": "https://www.browserbase.com/blog/introducing-the-stagehand-api", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Firecrawl", "url": "https://www.firecrawl.dev/", "fit": "Composite system" }, { "name": "Microsoft Playwright", "url": "https://playwright.dev/", "fit": "Composite system" }, { "name": "Skyvern", "url": "https://www.skyvern.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Steel", "url": "https://steel.dev/", "fit": "Composite system" }, { "name": "TinyFish", "url": "https://docs.tinyfish.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "pilot-published", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "browser" }, { "id": "webarena", "name": "WebArena", "href": "/leaderboard/webarena", "sourceUrl": "https://github.com/web-arena-x/webarena", "requirements": "Self-hosted sites plus BrowserGym or original harness.", "companies": [ { "name": "Automation Anywhere", "url": "https://www.automationanywhere.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Browser Use", "url": "https://browser-use.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Browserbase / Stagehand", "url": "https://www.browserbase.com/blog/introducing-the-stagehand-api", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Firecrawl", "url": "https://www.firecrawl.dev/", "fit": "Composite system" }, { "name": "Microsoft Playwright", "url": "https://playwright.dev/", "fit": "Composite system" }, { "name": "Skyvern", "url": "https://www.skyvern.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Steel", "url": "https://steel.dev/", "fit": "Composite system" }, { "name": "TinyFish", "url": "https://docs.tinyfish.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "UiPath", "url": "https://www.uipath.com/", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "browser" } ] }, { "id": "desktop-mobile", "name": "Desktop and mobile agents", "objective": "Application control, visual grounding and verified changes to application state.", "profile": "desktop", "profileName": "Desktop or emulator", "nextAction": "A qualified VM/emulator or grounding adapter, benchmark assets and native evaluator are required; the CPU container lane is insufficient.", "benchmarks": [ { "id": "osworld-verified", "name": "OSWorld-Verified", "href": "/leaderboard/osworld-verified", "sourceUrl": "https://github.com/xlang-ai/OSWorld", "requirements": "Desktop VM, screenshot/input adapter and pinned OS image.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Automation Anywhere", "url": "https://www.automationanywhere.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "UiPath", "url": "https://www.uipath.com/", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "desktop" }, { "id": "osworld-2-0", "name": "OSWorld 2.0", "href": "/leaderboard/osworld-2-0", "sourceUrl": "https://github.com/xlang-ai/OSWorld-V2", "requirements": "Desktop runtime and release-matched gated assets.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Automation Anywhere", "url": "https://www.automationanywhere.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "UiPath", "url": "https://www.uipath.com/", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "desktop" }, { "id": "screenspot-pro", "name": "ScreenSpot-Pro", "href": "/leaderboard/screenspot-pro", "sourceUrl": "https://huggingface.co/datasets/likaixin/ScreenSpot-Pro", "requirements": "Model returning screen coordinates for image and instruction.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Automation Anywhere", "url": "https://www.automationanywhere.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "UiPath", "url": "https://www.uipath.com/", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "desktop" }, { "id": "androidworld", "name": "AndroidWorld", "href": "/leaderboard/androidworld", "sourceUrl": "https://github.com/google-research/android_world", "requirements": "Android emulator, supported apps and action adapter.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Samsung", "url": "https://www.samsung.com/ae/news/local/galaxy-ai-expands-multi-agent-ecosystem-to-give-users-more-choice-and-flexibility/", "fit": "Product-specific adapter / qualified access" }, { "name": "UiPath", "url": "https://www.uipath.com/", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "desktop" } ] }, { "id": "coding", "name": "Coding agents", "objective": "Issue resolution, executable terminal work, regressions, time and cost.", "profile": "harbor", "profileName": "Harbor / CPU Linux", "nextAction": "Freeze the complete release, build task and verifier images, and qualify original positive and negative controls.", "benchmarks": [ { "id": "swe-bench-verified", "name": "SWE-bench Verified", "href": "/leaderboard/swe-bench-verified", "sourceUrl": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified", "requirements": "Repository-editing agent and isolated executable tests.", "companies": [ { "name": "All Hands / OpenHands", "url": "https://www.all-hands.dev/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Augment Code", "url": "https://www.augmentcode.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cline", "url": "https://cline.bot/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cognition / Devin", "url": "https://cognition.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cursor", "url": "https://cursor.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Factory", "url": "https://docs.factory.ai/cli/getting-started/overview", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "GitHub Copilot", "url": "https://github.com/features/copilot", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "JetBrains", "url": "https://www.jetbrains.com/junie/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Replit", "url": "https://replit.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "harbor" }, { "id": "swe-bench-pro", "name": "SWE-bench Pro", "href": "/leaderboard/swe-bench-pro", "sourceUrl": "https://github.com/scaleapi/SWE-bench_Pro-os", "requirements": "Repository agent and Pro-specific environments/tests.", "companies": [ { "name": "All Hands / OpenHands", "url": "https://www.all-hands.dev/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Augment Code", "url": "https://www.augmentcode.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cline", "url": "https://cline.bot/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cognition / Devin", "url": "https://cognition.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cursor", "url": "https://cursor.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Factory", "url": "https://docs.factory.ai/cli/getting-started/overview", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "GitHub Copilot", "url": "https://github.com/features/copilot", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "JetBrains", "url": "https://www.jetbrains.com/junie/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Replit", "url": "https://replit.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Sourcegraph", "url": "https://sourcegraph.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "harbor" }, { "id": "terminal-bench-2-1", "name": "Terminal-Bench 2.1", "href": "/leaderboard/terminal-bench-2-1", "sourceUrl": "https://www.tbench.ai/news/terminal-bench-2-1", "requirements": "Containerized command-line agent, Harbor and fixed resource budgets.", "companies": [ { "name": "All Hands / OpenHands", "url": "https://www.all-hands.dev/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Augment Code", "url": "https://www.augmentcode.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cline", "url": "https://cline.bot/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cognition / Devin", "url": "https://cognition.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cursor", "url": "https://cursor.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Factory", "url": "https://docs.factory.ai/cli/getting-started/overview", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "GitHub Copilot", "url": "https://github.com/features/copilot", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "JetBrains", "url": "https://www.jetbrains.com/junie/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Replit", "url": "https://replit.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "harbor" }, { "id": "terminal-bench-4-0", "name": "Terminal-Bench 4.0", "href": "/leaderboard/terminal-bench-4-0", "sourceUrl": "https://www.tbench.ai/news/terminal-bench-4-0", "requirements": "Harbor terminal agent; some tasks need GPUs or multiple containers.", "companies": [ { "name": "All Hands / OpenHands", "url": "https://www.all-hands.dev/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Augment Code", "url": "https://www.augmentcode.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cline", "url": "https://cline.bot/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cognition / Devin", "url": "https://cognition.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cursor", "url": "https://cursor.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Factory", "url": "https://docs.factory.ai/cli/getting-started/overview", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "GitHub Copilot", "url": "https://github.com/features/copilot", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "JetBrains", "url": "https://www.jetbrains.com/junie/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Replit", "url": "https://replit.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "harbor" } ] }, { "id": "customer-service", "name": "Customer-service agents", "objective": "Correct actions, policy adherence, completion and repeated-conversation consistency.", "profile": "simulation", "profileName": "Tool and conversation simulation", "nextAction": "Pin the simulator, policies, domain state and native evaluator; supply required service access and reset behavior.", "benchmarks": [ { "id": "bench-bench", "name": "τ-bench / τ²-bench", "href": "/leaderboard/bench-bench", "sourceUrl": "https://arxiv.org/abs/2506.07982", "requirements": "Agent linked to benchmark tools and a fixed user simulator.", "companies": [ { "name": "Ada", "url": "https://docs.ada.cx/docs/welcome", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Decagon", "url": "https://decagon.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Forethought", "url": "https://forethought.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Intercom", "url": "https://www.intercom.com/fin", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" }, { "name": "ServiceNow", "url": "https://www.servicenow.com/products/ai-agents.html", "fit": "Product-specific adapter / qualified access" }, { "name": "Sierra", "url": "https://sierra.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Zendesk", "url": "https://developer.zendesk.com/documentation/ai-agents/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "simulation" }, { "id": "bench-text-and-knowledge", "name": "τ³-bench: text and knowledge", "href": "/leaderboard/bench-text-and-knowledge", "sourceUrl": "https://github.com/sierra-research/tau2-bench", "requirements": "Tool adapter plus user simulation; retrieval setup for knowledge tasks.", "companies": [ { "name": "Ada", "url": "https://docs.ada.cx/docs/welcome", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Decagon", "url": "https://decagon.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Forethought", "url": "https://forethought.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "HubSpot", "url": "https://knowledge.hubspot.com/ai/understand-breeze", "fit": "Product-specific adapter / qualified access" }, { "name": "Intercom", "url": "https://www.intercom.com/fin", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" }, { "name": "ServiceNow", "url": "https://www.servicenow.com/products/ai-agents.html", "fit": "Product-specific adapter / qualified access" }, { "name": "Sierra", "url": "https://sierra.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Zendesk", "url": "https://developer.zendesk.com/documentation/ai-agents/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "simulation" }, { "id": "berkeley-function-calling-leaderboard-bfcl", "name": "Berkeley Function Calling Leaderboard (BFCL)", "href": "/leaderboard/berkeley-function-calling-leaderboard-bfcl", "sourceUrl": "https://gorilla.cs.berkeley.edu/leaderboard", "requirements": "Tool-call model adapter or instrumented agent interface.", "companies": [ { "name": "Alibaba Qwen", "url": "https://qwen.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cohere", "url": "https://cohere.com/", "fit": "Direct component API + selected system adapters" }, { "name": "Composio", "url": "https://composio.dev/", "fit": "Composite system" }, { "name": "DeepSeek", "url": "https://www.deepseek.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Meta", "url": "https://ai.meta.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "MiniMax", "url": "https://www.minimax.io/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" }, { "name": "Z.ai", "url": "https://z.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Zapier", "url": "https://zapier.com/agents", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "n8n", "url": "https://n8n.io/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "xAI", "url": "https://x.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "simulation" } ] }, { "id": "crm", "name": "CRM agents", "objective": "Correct record queries and mutations with an explicit schema and tool mapping.", "profile": "simulation", "profileName": "Tool and conversation simulation", "nextAction": "Pin the simulator, policies, domain state and native evaluator; supply required service access and reset behavior.", "benchmarks": [ { "id": "crmarena-crmarena-pro", "name": "CRMArena / CRMArena-Pro", "href": "/leaderboard/crmarena-crmarena-pro", "sourceUrl": "https://github.com/SalesforceAIResearch/CRMArena", "requirements": "Salesforce sandbox/API adapter.", "companies": [ { "name": "Decagon", "url": "https://decagon.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Gumloop", "url": "https://www.gumloop.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "HubSpot", "url": "https://knowledge.hubspot.com/ai/understand-breeze", "fit": "Product-specific adapter / qualified access" }, { "name": "Lindy", "url": "https://www.lindy.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" }, { "name": "Sierra", "url": "https://sierra.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "simulation" }, { "id": "bench-text-and-knowledge", "name": "τ³-bench: text and knowledge", "href": "/leaderboard/bench-text-and-knowledge", "sourceUrl": "https://github.com/sierra-research/tau2-bench", "requirements": "Tool adapter plus user simulation; retrieval setup for knowledge tasks.", "companies": [ { "name": "Ada", "url": "https://docs.ada.cx/docs/welcome", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Decagon", "url": "https://decagon.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Forethought", "url": "https://forethought.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "HubSpot", "url": "https://knowledge.hubspot.com/ai/understand-breeze", "fit": "Product-specific adapter / qualified access" }, { "name": "Intercom", "url": "https://www.intercom.com/fin", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" }, { "name": "ServiceNow", "url": "https://www.servicenow.com/products/ai-agents.html", "fit": "Product-specific adapter / qualified access" }, { "name": "Sierra", "url": "https://sierra.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Zendesk", "url": "https://developer.zendesk.com/documentation/ai-agents/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "simulation" } ] }, { "id": "enterprise", "name": "Enterprise workflow agents", "objective": "Multi-step completion across applications, documents and business records.", "profile": "world", "profileName": "Multi-service world / Harbor", "nextAction": "Qualify each world service, MCP endpoint, evidence mount and deterministic reset with its original grader.", "benchmarks": [ { "id": "workarena-workarena", "name": "WorkArena / WorkArena++", "href": "/leaderboard/workarena-workarena", "sourceUrl": "https://github.com/ServiceNow/WorkArena", "requirements": "ServiceNow instance and BrowserGym integration.", "companies": [ { "name": "Automation Anywhere", "url": "https://www.automationanywhere.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Browser Use", "url": "https://browser-use.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Browserbase / Stagehand", "url": "https://www.browserbase.com/blog/introducing-the-stagehand-api", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ServiceNow", "url": "https://www.servicenow.com/products/ai-agents.html", "fit": "Product-specific adapter / qualified access" }, { "name": "Skyvern", "url": "https://www.skyvern.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "UiPath", "url": "https://www.uipath.com/", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "world" }, { "id": "appworld", "name": "AppWorld", "href": "/leaderboard/appworld", "sourceUrl": "https://github.com/StonyBrookNLP/appworld", "requirements": "AppWorld runtime plus interactive code/tool agent.", "companies": [ { "name": "Composio", "url": "https://composio.dev/", "fit": "Composite system" }, { "name": "Gumloop", "url": "https://www.gumloop.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Lindy", "url": "https://www.lindy.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" }, { "name": "Zapier", "url": "https://zapier.com/agents", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "n8n", "url": "https://n8n.io/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "world" }, { "id": "theagentcompany", "name": "TheAgentCompany", "href": "/leaderboard/theagentcompany", "sourceUrl": "https://github.com/TheAgentCompany/TheAgentCompany", "requirements": "Multi-service company environment and agent tools.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Automation Anywhere", "url": "https://www.automationanywhere.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Gumloop", "url": "https://www.gumloop.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Lindy", "url": "https://www.lindy.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" }, { "name": "ServiceNow", "url": "https://www.servicenow.com/products/ai-agents.html", "fit": "Product-specific adapter / qualified access" }, { "name": "UiPath", "url": "https://www.uipath.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Zapier", "url": "https://zapier.com/agents", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "n8n", "url": "https://n8n.io/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "world" } ] }, { "id": "research", "name": "Search and research agents", "objective": "Difficult retrieval, supported answers, research quality and citation quality.", "profile": "browser", "profileName": "Browser / Harbor", "nextAction": "Prepare the complete task roster and benchmark-specific grading; live sites and self-hosted resettable sites require different environments.", "benchmarks": [ { "id": "browsecomp", "name": "BrowseComp", "href": "/leaderboard/browsecomp", "sourceUrl": "https://openai.com/index/browsecomp/", "requirements": "Search/browsing agent with fixed budgets.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Exa", "url": "https://exa.ai/", "fit": "Research endpoint or composite system" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Parallel", "url": "https://parallel.ai/", "fit": "Research endpoint or composite system" }, { "name": "Perplexity", "url": "https://www.perplexity.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Tavily", "url": "https://www.tavily.com/", "fit": "Research endpoint or composite system" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "You.com", "url": "https://you.com/", "fit": "Research endpoint or composite system" }, { "name": "Firecrawl", "url": "https://www.firecrawl.dev/", "fit": "Conditional on qualifying the research-preview Agent as an answer-producing system; raw scraping alone does not qualify." } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "browser" }, { "id": "gaia", "name": "GAIA", "href": "/leaderboard/gaia", "sourceUrl": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "requirements": "General agent with the required tool and media access.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Exa", "url": "https://exa.ai/", "fit": "Research endpoint or composite system" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Parallel", "url": "https://parallel.ai/", "fit": "Research endpoint or composite system" }, { "name": "Perplexity", "url": "https://www.perplexity.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Tavily", "url": "https://www.tavily.com/", "fit": "Research endpoint or composite system" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "You.com", "url": "https://you.com/", "fit": "Research endpoint or composite system" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "browser" }, { "id": "deepresearchbench", "name": "DeepResearchBench", "href": "/leaderboard/deepresearchbench", "sourceUrl": "https://github.com/Ayanami0730/deep_research_bench", "requirements": "Research system returning reports and citations; fixed judge pipeline.", "companies": [ { "name": "AlphaSense", "url": "https://www.alpha-sense.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Consensus", "url": "https://consensus.app/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Elicit", "url": "https://elicit.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Exa", "url": "https://exa.ai/", "fit": "Research endpoint or composite system" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Parallel", "url": "https://parallel.ai/", "fit": "Research endpoint or composite system" }, { "name": "Perplexity", "url": "https://www.perplexity.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Tavily", "url": "https://www.tavily.com/", "fit": "Research endpoint or composite system" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "You.com", "url": "https://you.com/", "fit": "Research endpoint or composite system" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "browser" }, { "id": "mind2web-2", "name": "Mind2Web 2", "href": "/leaderboard/mind2web-2", "sourceUrl": "https://github.com/OSU-NLP-Group/Mind2Web-2", "requirements": "Research agent and the benchmark judge workflow.", "companies": [ { "name": "Consensus", "url": "https://consensus.app/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Elicit", "url": "https://elicit.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Exa", "url": "https://exa.ai/", "fit": "Research endpoint or composite system" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Parallel", "url": "https://parallel.ai/", "fit": "Research endpoint or composite system" }, { "name": "Perplexity", "url": "https://www.perplexity.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Tavily", "url": "https://www.tavily.com/", "fit": "Research endpoint or composite system" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "You.com", "url": "https://you.com/", "fit": "Research endpoint or composite system" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "browser" } ] }, { "id": "retrieval", "name": "Embeddings and retrieval", "objective": "Document relevance and ranking; latency and throughput for complete pipelines.", "profile": "retrieval", "profileName": "Retrieval and structured evaluation", "nextAction": "Prepare the corpus and relevance labels, embedding/search or answer adapter, and the benchmark-native scoring pipeline.", "benchmarks": [ { "id": "mteb-mmteb", "name": "MTEB / MMTEB", "href": "/leaderboard/mteb-mmteb", "sourceUrl": "https://github.com/embeddings-benchmark/mteb", "requirements": "Embedding/reranking endpoint through the selected MTEB task collection.", "companies": [ { "name": "Alibaba Qwen", "url": "https://qwen.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cohere", "url": "https://cohere.com/", "fit": "Direct component API + selected system adapters" }, { "name": "Jina AI", "url": "https://jina.ai/", "fit": "Direct component API + selected system adapters" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "NVIDIA", "url": "https://www.nvidia.com/en-us/industries/robotics/", "fit": "Research / simulator adaptation" }, { "name": "Voyage AI", "url": "https://www.voyageai.com/", "fit": "Direct component API + selected system adapters" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "retrieval" }, { "id": "beir", "name": "BEIR", "href": "/leaderboard/beir", "sourceUrl": "https://github.com/beir-cellar/beir", "requirements": "Search/index pipeline with fixed corpus, chunking and embeddings.", "companies": [ { "name": "Cohere", "url": "https://cohere.com/", "fit": "Direct component API + selected system adapters" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Jina AI", "url": "https://jina.ai/", "fit": "Direct component API + selected system adapters" }, { "name": "Pinecone", "url": "https://www.pinecone.io/", "fit": "Composite system" }, { "name": "Qdrant", "url": "https://qdrant.tech/", "fit": "Composite system" }, { "name": "Vectara", "url": "https://www.vectara.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Voyage AI", "url": "https://www.voyageai.com/", "fit": "Direct component API + selected system adapters" }, { "name": "Weaviate", "url": "https://weaviate.io/", "fit": "Composite system" }, { "name": "Zilliz", "url": "https://zilliz.com/", "fit": "Composite system" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "retrieval" }, { "id": "bright", "name": "BRIGHT", "href": "/leaderboard/bright", "sourceUrl": "https://huggingface.co/datasets/mteb/BRIGHT", "requirements": "Retrieval/reranker or agentic retrieval system.", "companies": [ { "name": "Cohere", "url": "https://cohere.com/", "fit": "Direct component API + selected system adapters" }, { "name": "Exa", "url": "https://exa.ai/", "fit": "Research endpoint or composite system" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Jina AI", "url": "https://jina.ai/", "fit": "Direct component API + selected system adapters" }, { "name": "Pinecone", "url": "https://www.pinecone.io/", "fit": "Composite system" }, { "name": "Vectara", "url": "https://www.vectara.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Voyage AI", "url": "https://www.voyageai.com/", "fit": "Direct component API + selected system adapters" }, { "name": "Weaviate", "url": "https://weaviate.io/", "fit": "Composite system" }, { "name": "Qdrant", "url": "https://qdrant.tech/", "fit": "A configured retrieval pipeline must use the original corpus, exclusions and evaluator; disclose embeddings and any query reasoning." } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "retrieval" } ] }, { "id": "sql", "name": "SQL and business-analysis agents", "objective": "SQL execution correctness and multi-step business-analysis accuracy.", "profile": "data", "profileName": "Data and artifact evaluation", "nextAction": "Prepare databases/workbooks and original grading tools; external warehouses need their benchmark-specific access.", "benchmarks": [ { "id": "bird", "name": "BIRD", "href": "/leaderboard/bird", "sourceUrl": "https://bird-bench.github.io/", "requirements": "SQL generation, database access and query runner.", "companies": [ { "name": "Databricks", "url": "https://www.databricks.com/product/business-intelligence/ai-bi-genie", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hex", "url": "https://hex.tech/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Julius AI", "url": "https://julius.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "MotherDuck", "url": "https://motherduck.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Sigma", "url": "https://www.sigmacomputing.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Snowflake", "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-analyst", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ThoughtSpot", "url": "https://www.thoughtspot.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" }, { "id": "spider-2-0", "name": "Spider 2.0", "href": "/leaderboard/spider-2-0", "sourceUrl": "https://github.com/xlang-ai/Spider2", "requirements": "Warehouse or local database environment; some tracks require external accounts.", "companies": [ { "name": "Databricks", "url": "https://www.databricks.com/product/business-intelligence/ai-bi-genie", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hex", "url": "https://hex.tech/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Julius AI", "url": "https://julius.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "MotherDuck", "url": "https://motherduck.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Sigma", "url": "https://www.sigmacomputing.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Snowflake", "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-analyst", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ThoughtSpot", "url": "https://www.thoughtspot.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "dbt Labs", "url": "https://www.getdbt.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" }, { "id": "dabstep", "name": "DABstep", "href": "/leaderboard/dabstep", "sourceUrl": "https://huggingface.co/datasets/adyen/DABstep", "requirements": "Data agent with Python/shell and benchmark files.", "companies": [ { "name": "Databricks", "url": "https://www.databricks.com/product/business-intelligence/ai-bi-genie", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hex", "url": "https://hex.tech/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Julius AI", "url": "https://julius.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "MotherDuck", "url": "https://motherduck.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rogo", "url": "https://www.rogo.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Sigma", "url": "https://www.sigmacomputing.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Snowflake", "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-analyst", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ThoughtSpot", "url": "https://www.thoughtspot.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" } ] }, { "id": "spreadsheets", "name": "Spreadsheet agents", "objective": "Correct workbook artifacts, formulas and graded formatting.", "profile": "data", "profileName": "Data and artifact evaluation", "nextAction": "Prepare databases/workbooks and original grading tools; external warehouses need their benchmark-specific access.", "benchmarks": [ { "id": "spreadsheetbench-verified", "name": "SpreadsheetBench / Verified", "href": "/leaderboard/spreadsheetbench-verified", "sourceUrl": "https://github.com/RUCKBReasoning/SpreadsheetBench", "requirements": "Agent capable of editing spreadsheet files.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Julius AI", "url": "https://julius.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Microsoft Copilot", "url": "https://www.microsoft.com/en-us/microsoft-365-copilot", "fit": "Product-specific adapter / qualified access" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rogo", "url": "https://www.rogo.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Shortcut", "url": "https://shortcut.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hex", "url": "https://hex.tech/", "fit": "Conditional on a configured workflow accepting the required workbook and producing the native graded spreadsheet output." } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" }, { "id": "apex-agents-1-1", "name": "APEX-Agents 1.1", "href": "/leaderboard/apex-agents-1-1", "sourceUrl": "https://hub.harborframework.com/datasets/mercor/apex-agents-1-1", "requirements": "File and application tools plus the published rubric evaluator.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Harvey", "url": "https://www.harvey.ai/", "fit": "Product-specific adapter / qualified access" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Legora", "url": "https://legora.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Microsoft Copilot", "url": "https://www.microsoft.com/en-us/microsoft-365-copilot", "fit": "Product-specific adapter / qualified access" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rogo", "url": "https://www.rogo.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" } ] }, { "id": "documents", "name": "Document parsing and extraction", "objective": "Text, tables, reading order, formulas, fields and supporting evidence.", "profile": "document", "profileName": "Document evaluation", "nextAction": "Prepare source documents, parser interface and pinned text/table/layout evaluator images.", "benchmarks": [ { "id": "omnidocbench", "name": "OmniDocBench", "href": "/leaderboard/omnidocbench", "sourceUrl": "https://github.com/opendatalab/OmniDocBench", "requirements": "Document-to-structured-text/Markdown adapter.", "companies": [ { "name": "ABBYY", "url": "https://www.abbyy.com/", "fit": "Direct document API after output mapping" }, { "name": "Extend", "url": "https://www.extend.ai/", "fit": "Direct document API after output mapping" }, { "name": "Instabase", "url": "https://instabase.com/", "fit": "Direct document API after output mapping" }, { "name": "LlamaIndex", "url": "https://www.llamaindex.ai/", "fit": "Direct document API after output mapping" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Reducto", "url": "https://reducto.ai/", "fit": "Direct document API after output mapping" }, { "name": "Rossum", "url": "https://rossum.ai/", "fit": "Direct document API after output mapping" }, { "name": "Unstructured", "url": "https://unstructured.io/", "fit": "Direct document API after output mapping" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "document" }, { "id": "olmocr-bench", "name": "olmOCR-bench", "href": "/leaderboard/olmocr-bench", "sourceUrl": "https://huggingface.co/datasets/allenai/olmOCR-bench", "requirements": "Document parser producing normalized Markdown.", "companies": [ { "name": "ABBYY", "url": "https://www.abbyy.com/", "fit": "Direct document API after output mapping" }, { "name": "Extend", "url": "https://www.extend.ai/", "fit": "Direct document API after output mapping" }, { "name": "Instabase", "url": "https://instabase.com/", "fit": "Direct document API after output mapping" }, { "name": "LlamaIndex", "url": "https://www.llamaindex.ai/", "fit": "Direct document API after output mapping" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Reducto", "url": "https://reducto.ai/", "fit": "Direct document API after output mapping" }, { "name": "Rossum", "url": "https://rossum.ai/", "fit": "Direct document API after output mapping" }, { "name": "Unstructured", "url": "https://unstructured.io/", "fit": "Direct document API after output mapping" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "document" }, { "id": "parsebench", "name": "ParseBench", "href": "/leaderboard/parsebench", "sourceUrl": "https://huggingface.co/datasets/llamaindex/ParseBench", "requirements": "Document parser with the required output representations.", "companies": [ { "name": "ABBYY", "url": "https://www.abbyy.com/", "fit": "Direct document API after output mapping" }, { "name": "Extend", "url": "https://www.extend.ai/", "fit": "Direct document API after output mapping" }, { "name": "Instabase", "url": "https://instabase.com/", "fit": "Direct document API after output mapping" }, { "name": "LlamaIndex", "url": "https://www.llamaindex.ai/", "fit": "Direct document API after output mapping" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Reducto", "url": "https://reducto.ai/", "fit": "Direct document API after output mapping" }, { "name": "Rossum", "url": "https://rossum.ai/", "fit": "Direct document API after output mapping" }, { "name": "Unstructured", "url": "https://unstructured.io/", "fit": "Direct document API after output mapping" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "document" }, { "id": "extractbench", "name": "ExtractBench", "href": "/leaderboard/extractbench", "sourceUrl": "https://huggingface.co/datasets/llamaindex/ExtractBench", "requirements": "Structured extraction API returning schema-valid JSON and evidence.", "companies": [ { "name": "ABBYY", "url": "https://www.abbyy.com/", "fit": "Direct document API after output mapping" }, { "name": "Extend", "url": "https://www.extend.ai/", "fit": "Direct document API after output mapping" }, { "name": "Instabase", "url": "https://instabase.com/", "fit": "Direct document API after output mapping" }, { "name": "LlamaIndex", "url": "https://www.llamaindex.ai/", "fit": "Direct document API after output mapping" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Reducto", "url": "https://reducto.ai/", "fit": "Direct document API after output mapping" }, { "name": "Rossum", "url": "https://rossum.ai/", "fit": "Direct document API after output mapping" }, { "name": "Unstructured", "url": "https://unstructured.io/", "fit": "Direct document API after output mapping" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "document" } ] }, { "id": "legal", "name": "Legal research and contracts", "objective": "Reasoning, evidence retrieval, clause extraction and interpretation.", "profile": "retrieval", "profileName": "Retrieval and structured evaluation", "nextAction": "Prepare the corpus and relevance labels, embedding/search or answer adapter, and the benchmark-native scoring pipeline.", "benchmarks": [ { "id": "legalbench", "name": "LegalBench", "href": "/leaderboard/legalbench", "sourceUrl": "https://github.com/HazyResearch/legalbench", "requirements": "Text model or appropriately configured legal assistant.", "companies": [ { "name": "Harvey", "url": "https://www.harvey.ai/", "fit": "Product-specific adapter / qualified access" }, { "name": "Ironclad", "url": "https://ironcladapp.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Legora", "url": "https://legora.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "LexisNexis", "url": "https://www.lexisnexis.com/en-us/products/lexis-plus-ai.page", "fit": "Product-specific adapter / qualified access" }, { "name": "Luminance", "url": "https://www.luminance.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Thomson Reuters CoCounsel", "url": "https://legal.thomsonreuters.com/en/products/cocounsel", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "retrieval" }, { "id": "legalbench-rag", "name": "LegalBench-RAG", "href": "/leaderboard/legalbench-rag", "sourceUrl": "https://github.com/zeroentropy-ai/legalbenchrag", "requirements": "Retrieval pipeline over the provided legal corpus.", "companies": [ { "name": "Cohere", "url": "https://cohere.com/", "fit": "Direct component API + selected system adapters" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Harvey", "url": "https://www.harvey.ai/", "fit": "Product-specific adapter / qualified access" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Ironclad", "url": "https://ironcladapp.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Legora", "url": "https://legora.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "LexisNexis", "url": "https://www.lexisnexis.com/en-us/products/lexis-plus-ai.page", "fit": "Product-specific adapter / qualified access" }, { "name": "Luminance", "url": "https://www.luminance.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Thomson Reuters CoCounsel", "url": "https://legal.thomsonreuters.com/en/products/cocounsel", "fit": "Product-specific adapter / qualified access" }, { "name": "Voyage AI", "url": "https://www.voyageai.com/", "fit": "Direct component API + selected system adapters" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "retrieval" }, { "id": "cuad", "name": "CUAD", "href": "/leaderboard/cuad", "sourceUrl": "https://www.atticusprojectai.org/cuad", "requirements": "Contract extraction pipeline.", "companies": [ { "name": "Harvey", "url": "https://www.harvey.ai/", "fit": "Product-specific adapter / qualified access" }, { "name": "Instabase", "url": "https://instabase.com/", "fit": "Direct document API after output mapping" }, { "name": "Ironclad", "url": "https://ironcladapp.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Legora", "url": "https://legora.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "LexisNexis", "url": "https://www.lexisnexis.com/en-us/products/lexis-plus-ai.page", "fit": "Product-specific adapter / qualified access" }, { "name": "Luminance", "url": "https://www.luminance.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Reducto", "url": "https://reducto.ai/", "fit": "Direct document API after output mapping" }, { "name": "Thomson Reuters CoCounsel", "url": "https://legal.thomsonreuters.com/en/products/cocounsel", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "retrieval" }, { "id": "contractnli", "name": "ContractNLI", "href": "/leaderboard/contractnli", "sourceUrl": "https://stanfordnlp.github.io/contract-nli/", "requirements": "Contract text plus hypothesis; model outputs labels and evidence.", "companies": [ { "name": "Harvey", "url": "https://www.harvey.ai/", "fit": "Product-specific adapter / qualified access" }, { "name": "Ironclad", "url": "https://ironcladapp.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Legora", "url": "https://legora.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "LexisNexis", "url": "https://www.lexisnexis.com/en-us/products/lexis-plus-ai.page", "fit": "Product-specific adapter / qualified access" }, { "name": "Luminance", "url": "https://www.luminance.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Thomson Reuters CoCounsel", "url": "https://legal.thomsonreuters.com/en/products/cocounsel", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "retrieval" } ] }, { "id": "finance", "name": "Financial research", "objective": "Evidence-backed answers, numerical reasoning and professional deliverables.", "profile": "data", "profileName": "Data and artifact evaluation", "nextAction": "Prepare databases/workbooks and original grading tools; external warehouses need their benchmark-specific access.", "benchmarks": [ { "id": "financebench", "name": "FinanceBench", "href": "/leaderboard/financebench", "sourceUrl": "https://github.com/patronus-ai/financebench", "requirements": "Document retrieval and financial QA pipeline.", "companies": [ { "name": "AlphaSense", "url": "https://www.alpha-sense.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rogo", "url": "https://www.rogo.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Thomson Reuters CoCounsel", "url": "https://legal.thomsonreuters.com/en/products/cocounsel", "fit": "Product-specific adapter / qualified access" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" }, { "id": "finqa-convfinqa", "name": "FinQA / ConvFinQA", "href": "/leaderboard/finqa-convfinqa", "sourceUrl": "https://github.com/czyssrs/FinQA", "requirements": "Financial reasoning model with calculator/code tools where allowed.", "companies": [ { "name": "AlphaSense", "url": "https://www.alpha-sense.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Databricks", "url": "https://www.databricks.com/product/business-intelligence/ai-bi-genie", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rogo", "url": "https://www.rogo.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Snowflake", "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-analyst", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" }, { "id": "apex-agents-1-1", "name": "APEX-Agents 1.1", "href": "/leaderboard/apex-agents-1-1", "sourceUrl": "https://hub.harborframework.com/datasets/mercor/apex-agents-1-1", "requirements": "File and application tools plus the published rubric evaluator.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Harvey", "url": "https://www.harvey.ai/", "fit": "Product-specific adapter / qualified access" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Legora", "url": "https://legora.com/", "fit": "Product-specific adapter / qualified access" }, { "name": "Microsoft Copilot", "url": "https://www.microsoft.com/en-us/microsoft-365-copilot", "fit": "Product-specific adapter / qualified access" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rogo", "url": "https://www.rogo.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Writer", "url": "https://writer.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "data" } ] }, { "id": "asr", "name": "Speech recognition", "objective": "Transcription across languages and domains; diarization scored separately.", "profile": "audio", "profileName": "Speech input/output", "nextAction": "Qualify an ASR or TTS endpoint, source audio and native scoring; reference-speaker and language conditions must be preserved.", "benchmarks": [ { "id": "open-asr-leaderboard-esb", "name": "Open ASR Leaderboard / ESB", "href": "/leaderboard/open-asr-leaderboard-esb", "sourceUrl": "https://huggingface.co/blog/open-asr-leaderboard", "requirements": "Speech-to-text API or local model.", "companies": [ { "name": "AssemblyAI", "url": "https://www.assemblyai.com/", "fit": "Direct ASR API" }, { "name": "Cartesia", "url": "https://docs.cartesia.ai/api-reference/voices/clone", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Deepgram", "url": "https://developers.deepgram.com/docs/tts-models", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ElevenLabs", "url": "https://elevenlabs.io/docs/eleven-api/concepts/voice-cloning", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Gladia", "url": "https://www.gladia.io/", "fit": "Direct ASR API" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rev", "url": "https://www.rev.com/", "fit": "Direct ASR API" }, { "name": "Soniox", "url": "https://soniox.com/", "fit": "Direct ASR API" }, { "name": "Speechmatics", "url": "https://www.speechmatics.com/", "fit": "Direct ASR API" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "audio" }, { "id": "fleurs", "name": "FLEURS", "href": "/leaderboard/fleurs", "sourceUrl": "https://huggingface.co/datasets/google/fleurs", "requirements": "ASR or speech system supporting the evaluated languages.", "companies": [ { "name": "AssemblyAI", "url": "https://www.assemblyai.com/", "fit": "Direct ASR API" }, { "name": "Cartesia", "url": "https://docs.cartesia.ai/api-reference/voices/clone", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Deepgram", "url": "https://developers.deepgram.com/docs/tts-models", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ElevenLabs", "url": "https://elevenlabs.io/docs/eleven-api/concepts/voice-cloning", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Gladia", "url": "https://www.gladia.io/", "fit": "Direct ASR API" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Meta", "url": "https://ai.meta.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rev", "url": "https://www.rev.com/", "fit": "Direct ASR API" }, { "name": "Soniox", "url": "https://soniox.com/", "fit": "Direct ASR API" }, { "name": "Speechmatics", "url": "https://www.speechmatics.com/", "fit": "Direct ASR API" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "audio" }, { "id": "earnings-22", "name": "Earnings-22", "href": "/leaderboard/earnings-22", "sourceUrl": "https://github.com/revdotcom/speech-datasets/tree/main/earnings22", "requirements": "ASR system handling long audio.", "companies": [ { "name": "AlphaSense", "url": "https://www.alpha-sense.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "AssemblyAI", "url": "https://www.assemblyai.com/", "fit": "Direct ASR API" }, { "name": "Cartesia", "url": "https://docs.cartesia.ai/api-reference/voices/clone", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Deepgram", "url": "https://developers.deepgram.com/docs/tts-models", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Fireflies", "url": "https://fireflies.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Gladia", "url": "https://www.gladia.io/", "fit": "Direct ASR API" }, { "name": "Otter", "url": "https://otter.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rev", "url": "https://www.rev.com/", "fit": "Direct ASR API" }, { "name": "Soniox", "url": "https://soniox.com/", "fit": "Direct ASR API" }, { "name": "Speechmatics", "url": "https://www.speechmatics.com/", "fit": "Direct ASR API" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "audio" }, { "id": "ami-meeting-corpus", "name": "AMI meeting corpus", "href": "/leaderboard/ami-meeting-corpus", "sourceUrl": "https://groups.inf.ed.ac.uk/ami/corpus/", "requirements": "Meeting transcription and speaker-diarization pipeline.", "companies": [ { "name": "AssemblyAI", "url": "https://www.assemblyai.com/", "fit": "Direct ASR API" }, { "name": "Deepgram", "url": "https://developers.deepgram.com/docs/tts-models", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Fireflies", "url": "https://fireflies.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Gladia", "url": "https://www.gladia.io/", "fit": "Direct ASR API" }, { "name": "Otter", "url": "https://otter.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rev", "url": "https://www.rev.com/", "fit": "Direct ASR API" }, { "name": "Speechmatics", "url": "https://www.speechmatics.com/", "fit": "Direct ASR API" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "audio" } ] }, { "id": "tts", "name": "Speech synthesis and cloning", "objective": "Intelligibility and reference-speaker similarity; separate listening and latency tests.", "profile": "audio", "profileName": "Speech input/output", "nextAction": "Qualify an ASR or TTS endpoint, source audio and native scoring; reference-speaker and language conditions must be preserved.", "benchmarks": [ { "id": "seed-tts-eval", "name": "Seed-TTS-eval", "href": "/leaderboard/seed-tts-eval", "sourceUrl": "https://github.com/BytedanceSpeech/seed-tts-eval", "requirements": "Reference-conditioned TTS API; generate WAV/audio for the prescribed text.", "companies": [ { "name": "Cartesia", "url": "https://docs.cartesia.ai/api-reference/voices/clone", "fit": "TTS API; full cloning protocol conditional" }, { "name": "ElevenLabs", "url": "https://elevenlabs.io/docs/eleven-api/concepts/voice-cloning", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Fish Audio", "url": "https://docs.fish.audio/api-reference/sdk/python/resources", "fit": "TTS API; full cloning protocol conditional" }, { "name": "HeyGen", "url": "https://www.heygen.com/tool/ai-voice-cloning", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Murf AI", "url": "https://murf.ai/api/docs/api-reference/voice-cloning/create", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Resemble AI", "url": "https://www.resemble.ai/", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Speechify", "url": "https://docs.speechify.ai/tts/guides/voice-cloning", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Synthesia", "url": "https://docs.synthesia.io/docs/custom-voices", "fit": "TTS API; full cloning protocol conditional" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "audio" } ] }, { "id": "voice", "name": "Conversational voice agents", "objective": "Spoken task completion, tool use, interruptions and response latency.", "profile": "voice", "profileName": "Streaming voice simulation", "nextAction": "Qualify streaming audio, interruption timing, tool/user simulation and separate task, ASR and TTS scores.", "benchmarks": [ { "id": "voice-full-duplex-voice", "name": "τ-Voice / τ³ full-duplex voice", "href": "/leaderboard/voice-full-duplex-voice", "sourceUrl": "https://arxiv.org/abs/2603.13686", "requirements": "Native real-time audio adapter and benchmark tool/user simulation.", "companies": [ { "name": "Bland AI", "url": "https://www.bland.ai/", "fit": "Full voice-stack adapter" }, { "name": "Cartesia", "url": "https://docs.cartesia.ai/api-reference/voices/clone", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Cresta", "url": "https://cresta.com/", "fit": "Full voice-stack adapter" }, { "name": "Decagon", "url": "https://decagon.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Deepgram", "url": "https://developers.deepgram.com/docs/tts-models", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ElevenLabs", "url": "https://elevenlabs.io/docs/eleven-api/concepts/voice-cloning", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Hume AI", "url": "https://www.hume.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Parloa", "url": "https://www.parloa.com/", "fit": "Full voice-stack adapter" }, { "name": "PolyAI", "url": "https://poly.ai/", "fit": "Full voice-stack adapter" }, { "name": "Retell AI", "url": "https://www.retellai.com/", "fit": "Full voice-stack adapter" }, { "name": "Sierra", "url": "https://sierra.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Vapi", "url": "https://vapi.ai/", "fit": "Full voice-stack adapter" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "voice" }, { "id": "voicebench", "name": "VoiceBench", "href": "/leaderboard/voicebench", "sourceUrl": "https://github.com/MatthewCYM/VoiceBench", "requirements": "Voice assistant with audio input and benchmark-compatible answers.", "companies": [ { "name": "Bland AI", "url": "https://www.bland.ai/", "fit": "Full voice-stack adapter" }, { "name": "Cartesia", "url": "https://docs.cartesia.ai/api-reference/voices/clone", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Cresta", "url": "https://cresta.com/", "fit": "Full voice-stack adapter" }, { "name": "Deepgram", "url": "https://developers.deepgram.com/docs/tts-models", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "ElevenLabs", "url": "https://elevenlabs.io/docs/eleven-api/concepts/voice-cloning", "fit": "TTS API; full cloning protocol conditional" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hume AI", "url": "https://www.hume.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Parloa", "url": "https://www.parloa.com/", "fit": "Full voice-stack adapter" }, { "name": "PolyAI", "url": "https://poly.ai/", "fit": "Full voice-stack adapter" }, { "name": "Retell AI", "url": "https://www.retellai.com/", "fit": "Full voice-stack adapter" }, { "name": "Vapi", "url": "https://vapi.ai/", "fit": "Full voice-stack adapter" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "voice" } ] }, { "id": "images", "name": "Image generation", "objective": "Prompt adherence, counts, attributes and spatial relationships.", "profile": "generation", "profileName": "Media generation and scoring", "nextAction": "Qualify image/video generation and benchmark-native evaluator images; GPU requirements need a separately bounded execution profile.", "benchmarks": [ { "id": "geneval", "name": "GenEval", "href": "/leaderboard/geneval", "sourceUrl": "https://github.com/djghosh13/geneval", "requirements": "Image generation API with fixed prompts, seeds and sample counts.", "companies": [ { "name": "Adobe", "url": "https://www.adobe.com/products/firefly.html", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Black Forest Labs", "url": "https://bfl.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Canva", "url": "https://www.canva.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Ideogram", "url": "https://ideogram.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Midjourney", "url": "https://www.midjourney.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Stability AI", "url": "https://stability.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "generation" }, { "id": "dpg-bench", "name": "DPG-Bench", "href": "/leaderboard/dpg-bench", "sourceUrl": "https://github.com/TencentQQGYLab/ELLA", "requirements": "Image generator and fixed vision-language evaluator.", "companies": [ { "name": "Adobe", "url": "https://www.adobe.com/products/firefly.html", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Black Forest Labs", "url": "https://bfl.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Canva", "url": "https://www.canva.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Ideogram", "url": "https://ideogram.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Midjourney", "url": "https://www.midjourney.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Stability AI", "url": "https://stability.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "generation" }, { "id": "t2i-compbench-t2i-compbench", "name": "T2I-CompBench / T2I-CompBench++", "href": "/leaderboard/t2i-compbench-t2i-compbench", "sourceUrl": "https://github.com/Karine-Huang/T2I-CompBench", "requirements": "Text-to-image generator and published evaluators.", "companies": [ { "name": "Adobe", "url": "https://www.adobe.com/products/firefly.html", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Black Forest Labs", "url": "https://bfl.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Canva", "url": "https://www.canva.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Ideogram", "url": "https://ideogram.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Midjourney", "url": "https://www.midjourney.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Stability AI", "url": "https://stability.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "generation" } ] }, { "id": "video", "name": "Video generation", "objective": "Temporal consistency, quality, motion and prompt alignment.", "profile": "generation", "profileName": "Media generation and scoring", "nextAction": "Qualify image/video generation and benchmark-native evaluator images; GPU requirements need a separately bounded execution profile.", "benchmarks": [ { "id": "vbench-vbench-2-0", "name": "VBench / VBench 2.0", "href": "/leaderboard/vbench-vbench-2-0", "sourceUrl": "https://github.com/Vchitect/VBench", "requirements": "Video model with matched prompts, duration and resolution.", "companies": [ { "name": "Adobe", "url": "https://www.adobe.com/products/firefly.html", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Kling", "url": "https://klingai.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Luma", "url": "https://lumalabs.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "MiniMax", "url": "https://www.minimax.io/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Pika", "url": "https://pika.art/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Runway", "url": "https://runwayml.com/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "generation" } ] }, { "id": "multimodal", "name": "Multimodal understanding", "objective": "Reasoning and understanding over images and video.", "profile": "multimodal", "profileName": "Image and video understanding", "nextAction": "Prepare licensed media, modality-capable model input and benchmark-native scoring with the original sampling protocol.", "benchmarks": [ { "id": "mmmu-pro", "name": "MMMU-Pro", "href": "/leaderboard/mmmu-pro", "sourceUrl": "https://huggingface.co/datasets/MMMU/MMMU_Pro", "requirements": "Vision-language model.", "companies": [ { "name": "Alibaba Qwen", "url": "https://qwen.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cohere", "url": "https://cohere.com/", "fit": "Direct component API + selected system adapters" }, { "name": "DeepSeek", "url": "https://www.deepseek.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Meta", "url": "https://ai.meta.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "MiniMax", "url": "https://www.minimax.io/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Twelve Labs", "url": "https://www.twelvelabs.io/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Z.ai", "url": "https://z.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "xAI", "url": "https://x.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "multimodal" }, { "id": "mathvista", "name": "MathVista", "href": "/leaderboard/mathvista", "sourceUrl": "https://huggingface.co/datasets/AI4Math/MathVista", "requirements": "Image-and-text reasoning model.", "companies": [ { "name": "Alibaba Qwen", "url": "https://qwen.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Cohere", "url": "https://cohere.com/", "fit": "Direct component API + selected system adapters" }, { "name": "DeepSeek", "url": "https://www.deepseek.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Hebbia", "url": "https://www.hebbia.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Meta", "url": "https://ai.meta.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "MiniMax", "url": "https://www.minimax.io/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Mistral", "url": "https://mistral.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Rogo", "url": "https://www.rogo.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Z.ai", "url": "https://z.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "xAI", "url": "https://x.ai/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "multimodal" }, { "id": "video-mme-video-mme-v2", "name": "Video-MME / Video-MME v2", "href": "/leaderboard/video-mme-video-mme-v2", "sourceUrl": "https://huggingface.co/datasets/MME-Benchmarks/Video-MME-v2", "requirements": "Video model with declared frame sampling, audio and subtitle access.", "companies": [ { "name": "Alibaba Qwen", "url": "https://qwen.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Google DeepMind", "url": "https://deepmind.google/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Meta", "url": "https://ai.meta.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Runway", "url": "https://runwayml.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Twelve Labs", "url": "https://www.twelvelabs.io/", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "multimodal" } ] }, { "id": "security", "name": "Agent security", "objective": "Prompt-injection resistance while preserving legitimate task success.", "profile": "simulation", "profileName": "Tool and conversation simulation", "nextAction": "Pin the simulator, policies, domain state and native evaluator; supply required service access and reset behavior.", "benchmarks": [ { "id": "agentdojo", "name": "AgentDojo", "href": "/leaderboard/agentdojo", "sourceUrl": "https://github.com/ethz-spylab/agentdojo", "requirements": "Agent + defensive product in the same benchmark workflow.", "companies": [ { "name": "Anthropic", "url": "https://www.anthropic.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Browser Use", "url": "https://browser-use.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Glean", "url": "https://www.glean.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "HiddenLayer", "url": "https://hiddenlayer.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Lakera", "url": "https://www.lakera.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "OpenAI", "url": "https://openai.com/index/browsecomp/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Prompt Security", "url": "https://www.prompt.security/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Protect AI", "url": "https://protectai.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Salesforce", "url": "https://www.salesforce.com/agentforce/", "fit": "Product-specific adapter / qualified access" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "simulation" }, { "id": "injecagent", "name": "InjecAgent", "href": "/leaderboard/injecagent", "sourceUrl": "https://github.com/uiuc-kang-lab/InjecAgent", "requirements": "Tool-integrated model/agent adapter.", "companies": [ { "name": "Composio", "url": "https://composio.dev/", "fit": "Composite system" }, { "name": "HiddenLayer", "url": "https://hiddenlayer.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Lakera", "url": "https://www.lakera.ai/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Prompt Security", "url": "https://www.prompt.security/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Protect AI", "url": "https://protectai.com/", "fit": "Capability-aligned; adapter/access to qualify" }, { "name": "Zapier", "url": "https://zapier.com/agents", "fit": "Capability-aligned; adapter/access to qualify" } ], "status": "qualification-required", "fullRun": false, "gkeTarget": "bf-benchmarks", "executionProfile": "simulation" } ] } ], "portfolio": [ { "id": "counselbench-100", "name": "CounselBench-100", "href": "/benchmarks/counselbench-100", "sourceUrl": "https://github.com/blobfishai/legal-agent-simulation/tree/master/benchmark/counselbench100", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "salesbench-100", "name": "SalesBench-100", "href": "/benchmarks/salesbench-100", "sourceUrl": "https://github.com/blobfishai/sales-agent-simulation", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "devopsbench-100", "name": "DevOpsBench-100", "href": "/benchmarks/devopsbench-100", "sourceUrl": "https://github.com/blobfishai/software-devops", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "full-package-registered", "blocker": "All 100 original tasks and immutable runtime images are registered. Complete model attempts and grading remain required; earlier ten-task controls and pilots retain their original scope.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "ledgerbench-100", "name": "LedgerBench-100", "href": "/benchmarks/ledgerbench-100", "sourceUrl": "https://github.com/blobfishai/finance-world", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "factorybench-100", "name": "FactoryBench-100", "href": "/benchmarks/factorybench-100", "sourceUrl": "https://github.com/blobfishai/factory-agent-simulation", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "published" }, { "id": "semikongbench-100", "name": "SemiKongBench-100", "href": "/benchmarks/semikongbench-100", "sourceUrl": "https://github.com/blobfishai/semiconductor-agent-simulation", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "dealbench-100", "name": "DealBench-100", "href": "/benchmarks/dealbench-100", "sourceUrl": "https://github.com/blobfishai/blobfishai/tree/main/benchmark/dealbench100", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "erpbench-100", "name": "ERPBench-100", "href": "/benchmarks/erpbench-100", "sourceUrl": "https://github.com/blobfishai/erp-agent-simulation/tree/main/benchmark/erpbench100", "kind": "Original benchmark", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "arc-crm-6", "name": "Arc CRM 6", "href": "/benchmarks/arc-crm-6", "sourceUrl": "https://github.com/blobfishai/arc-crm-agent-simulation/tree/090d4028e485d21190588006645a13e125aef130", "kind": "Original benchmark", "tasks": 6, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Prepare the exact complete published release, immutable images and original verifier; qualify world services, MCP tools, evidence mounts and native task timeouts.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "awaiting-first-model-run" }, { "id": "filingbench-50", "name": "FilingBench-50", "href": "/benchmarks/filingbench-50", "sourceUrl": "https://github.com/vals-ai/finance-agent", "kind": "Companion world", "tasks": 4, "upstreamTasks": 50, "profile": "world", "status": "qualification-required", "blocker": "The four-task companion world needs its own prepared GKE package and verifier qualification. Its upstream benchmark remains a separate run.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "baseline-evidence-live" }, { "id": "servicebench-375", "name": "ServiceBench-375", "href": "/benchmarks/servicebench-375", "sourceUrl": "https://github.com/sierra-research/tau2-bench", "kind": "Companion world", "tasks": 4, "upstreamTasks": 375, "profile": "world", "status": "qualification-required", "blocker": "The four-task companion world needs its own prepared GKE package and verifier qualification. Its upstream benchmark remains a separate run.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "baseline-evidence-live" }, { "id": "discoverybench-102", "name": "DiscoveryBench-102", "href": "/benchmarks/discoverybench-102", "sourceUrl": "https://github.com/OSU-NLP-Group/ScienceAgentBench", "kind": "Companion world", "tasks": 4, "upstreamTasks": 102, "profile": "world", "status": "qualification-required", "blocker": "The four-task companion world needs its own prepared GKE package and verifier qualification. Its upstream benchmark remains a separate run.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "baseline-evidence-live" }, { "id": "defensebench-100", "name": "DefenseBench-100", "href": "/benchmarks/defensebench-100", "sourceUrl": "https://hub.harborframework.com/datasets/polyvorlabs/cyberdefense-bench", "kind": "Companion world", "tasks": 4, "upstreamTasks": 100, "profile": "world", "status": "qualification-required", "blocker": "The four-task companion world needs its own prepared GKE package and verifier qualification. Its upstream benchmark remains a separate run.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "baseline-evidence-live" }, { "id": "opsbench-14", "name": "OpsBench-14", "href": "/benchmarks/opsbench-14", "sourceUrl": "https://github.com/devrev/enterprise-bench", "kind": "Companion world", "tasks": 4, "upstreamTasks": 14, "profile": "world", "status": "qualification-required", "blocker": "The four-task companion world needs its own prepared GKE package and verifier qualification. Its upstream benchmark remains a separate run.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "baseline-evidence-live" }, { "id": "repobench-500", "name": "RepoBench-500", "href": "/benchmarks/repobench-500", "sourceUrl": "https://github.com/SWE-bench/SWE-bench", "kind": "Companion world", "tasks": 4, "upstreamTasks": 500, "profile": "world", "status": "qualification-required", "blocker": "The four-task companion world needs its own prepared GKE package and verifier qualification. Its upstream benchmark remains a separate run.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "baseline-evidence-live" }, { "id": "northstar-composite", "name": "Northstar Composite Ops", "href": "/w/frontier-lab-composite-ops-v1", "sourceUrl": "https://github.com/blobfishai/blobfishai/tree/main/demo/frontier-lab-composite-world", "kind": "World proof", "tasks": 100, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Full 100-task native Harbor adapter merged in #4592. All original local controls pass; the prepared image still requires live GKE qualification and a complete measured model run. The separate customer release keeps its own identity.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "world-proof" }, { "id": "hubbench", "name": "HubBench", "href": "/benchmarks/hubbench", "sourceUrl": "https://github.com/blobfishai/blobfishai/tree/main/benchmark/hubbench", "kind": "13-family suite", "tasks": 104, "upstreamTasks": null, "profile": "world", "status": "qualification-required", "blocker": "Qualify the complete 104-task release, multiple MCP endpoints, named internal networks and original 1,200-second task budgets.", "gkeTarget": "bf-benchmarks", "fullRun": false, "existingEvidence": "oracle-controls" } ] }