{
  "observedAt": "2026-09-16T04:23:21.089063+00:00",
  "registryUrl": "https://blobfish.ai/api/v1/agent-reports/benchmarks",
  "cluster": "bf-benchmarks",
  "location": "us-central1-a",
  "format": "Harbor and HUD",
  "benchmarks": [
    {
      "id": "devopsbench-100",
      "name": "DevOpsBench-100",
      "version": "3.2.7:revision8:pilot-1",
      "digest": "09fa6e0c38eedf4223f9262166bfac29c9497080e2b575374b0474ca9b7ecfb2",
      "format": "harbor",
      "description": "Ten matching tasks from the exact published revision 8 (2 source-heldout, 8 source-train). Original main/world images and original deterministic state verifiers. Pilot time/command budgets are recorded per run.",
      "tasks": 10,
      "official_tasks": 100,
      "control": false,
      "scaffolds": [
        "harbor-terminus-2",
        "harbor-codex",
        "harbor-claude-code",
        "harbor-codex-subscription",
        "harbor-claude-subscription",
        "harbor-deepseek"
      ],
      "task_ids": [
        "dob100-010-detect-status-page-recurrence",
        "dob100-014-localize-checkout-latency",
        "dob100-035-port-close-backlog-issues",
        "dob100-036-port-close-blocked-issues",
        "dob100-049-payments-retry",
        "dob100-062-w6-copy-13",
        "dob100-076-gateway-pool-reuse",
        "dob100-081-backorders",
        "dob100-085-notification-templates",
        "dob100-089-rcn-customer-facing-incidents"
      ],
      "grading": {
        "native": 10,
        "rubric": 0,
        "vision": 0
      },
      "grading_engines": []
    },
    {
      "id": "humaneval",
      "name": "HumanEval",
      "version": "6d43fb980f9f:pilot-1",
      "digest": "47adf872a27bf6ae875d4ac4d874e151532770864c2affcaff4fd586d1ca87f6",
      "format": "harbor",
      "description": "Measured ten-task sample from 164 public evaluation tasks. One attempt per task with a common terminal scaffold; original tests, RNG seed 0. This is an agentic evaluation, not the original code-only sampling protocol.",
      "tasks": 10,
      "official_tasks": 164,
      "control": false,
      "scaffolds": [
        "harbor-terminus-2",
        "harbor-codex-subscription",
        "harbor-claude-subscription",
        "harbor-deepseek"
      ],
      "task_ids": [
        "humaneval-2",
        "humaneval-39",
        "humaneval-48",
        "humaneval-50",
        "humaneval-61",
        "humaneval-64",
        "humaneval-75",
        "humaneval-79",
        "humaneval-100",
        "humaneval-136"
      ],
      "grading": {
        "native": 10,
        "rubric": 0,
        "vision": 0
      },
      "grading_engines": []
    },
    {
      "id": "mbpp",
      "name": "MBPP",
      "version": "f46ca8374b4c:pilot-1",
      "digest": "28cbcf6c4d6a792dd22b7441b347c709e292c4998ec7e3bb9e4e2a97d1ca97eb",
      "format": "harbor",
      "description": "Measured ten-task sample from 500 public evaluation tasks. One attempt per task with a common terminal scaffold; original tests, RNG seed 0. This is an agentic evaluation, not the original code-only sampling protocol.",
      "tasks": 10,
      "official_tasks": 500,
      "control": false,
      "scaffolds": [
        "harbor-terminus-2",
        "harbor-codex-subscription",
        "harbor-claude-subscription",
        "harbor-deepseek"
      ],
      "task_ids": [
        "mbpp-14",
        "mbpp-34",
        "mbpp-42",
        "mbpp-182",
        "mbpp-273",
        "mbpp-322",
        "mbpp-352",
        "mbpp-398",
        "mbpp-496",
        "mbpp-505"
      ],
      "grading": {
        "native": 10,
        "rubric": 0,
        "vision": 0
      },
      "grading_engines": []
    }
  ],
  "qualification": [
    {
      "benchmark": "humaneval",
      "date": "2026-09-16T02:32:55.804209+00:00",
      "runId": "br-d0b7ecaf6d13fb49bfb18402049097a6",
      "status": "completed",
      "referencePassed": 10,
      "referenceTotal": 10,
      "noOpRejected": 10,
      "noOpTotal": 10,
      "outcomes": [
        {
          "task": "humaneval-2",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-39",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-48",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-50",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-61",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-64",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-75",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-79",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-100",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-136",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-2",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-39",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-48",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-50",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-61",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-64",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-75",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-79",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-100",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "humaneval-136",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        }
      ]
    },
    {
      "benchmark": "mbpp",
      "date": "2026-09-16T02:38:02.928380+00:00",
      "runId": "br-26c40d4c46ca67a16d5f1243d0e8e2c5",
      "status": "completed",
      "referencePassed": 10,
      "referenceTotal": 10,
      "noOpRejected": 10,
      "noOpTotal": 10,
      "outcomes": [
        {
          "task": "mbpp-14",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-34",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-42",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-182",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-273",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-322",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-352",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-398",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-496",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-505",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-14",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-34",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-42",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-182",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-273",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-322",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-352",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-398",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-496",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "mbpp-505",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        }
      ]
    },
    {
      "benchmark": "devops-first-two",
      "date": "2026-09-16T02:36:06.538465+00:00",
      "runId": "br-60b5f869964198d5da3bb6e3cc773ceb",
      "status": "completed",
      "referencePassed": 2,
      "referenceTotal": 2,
      "noOpRejected": 2,
      "noOpTotal": 2,
      "outcomes": [
        {
          "task": "dob100-010-detect-status-page-recurrence",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-014-localize-checkout-latency",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-010-detect-status-page-recurrence",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-014-localize-checkout-latency",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        }
      ]
    },
    {
      "benchmark": "devops-remaining-eight",
      "date": "2026-09-16T02:39:39.563909+00:00",
      "runId": "br-19cc7a102f92fa8d08c82a598e14796e",
      "status": "completed",
      "referencePassed": 8,
      "referenceTotal": 8,
      "noOpRejected": 8,
      "noOpTotal": 8,
      "outcomes": [
        {
          "task": "dob100-035-port-close-backlog-issues",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-036-port-close-blocked-issues",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-049-payments-retry",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-062-w6-copy-13",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-076-gateway-pool-reuse",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-081-backorders",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-085-notification-templates",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-089-rcn-customer-facing-incidents",
          "agent": "oracle",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-035-port-close-backlog-issues",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-036-port-close-blocked-issues",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-049-payments-retry",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-062-w6-copy-13",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-076-gateway-pool-reuse",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-081-backorders",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-085-notification-templates",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        },
        {
          "task": "dob100-089-rcn-customer-facing-incidents",
          "agent": "nop",
          "status": "completed",
          "native": false,
          "rubric": null,
          "vision": null
        }
      ]
    },
    {
      "benchmark": "claude-subscription",
      "date": "2026-09-16T02:31:12.359498+00:00",
      "runId": "br-9c0a65bdc49ba5b10b7ba28c78e67dd5",
      "status": "completed",
      "referencePassed": 0,
      "referenceTotal": 0,
      "noOpRejected": 0,
      "noOpTotal": 0,
      "outcomes": [
        {
          "task": "answer-42",
          "agent": "claude",
          "status": "completed",
          "native": true,
          "rubric": null,
          "vision": null
        }
      ]
    },
    {
      "benchmark": "codex-rubric",
      "date": "2026-09-16T02:37:52.847636+00:00",
      "runId": "br-834551ee19d75311b1ecd354e87d1e9e",
      "status": "completed",
      "referencePassed": 0,
      "referenceTotal": 0,
      "noOpRejected": 0,
      "noOpTotal": 0,
      "outcomes": [
        {
          "task": "answer-42",
          "agent": "positive",
          "status": "completed",
          "native": true,
          "rubric": true,
          "vision": null
        },
        {
          "task": "answer-42",
          "agent": "negative",
          "status": "completed",
          "native": false,
          "rubric": false,
          "vision": null
        }
      ]
    }
  ],
  "qualificationSource": "https://github.com/blobfishai/blobfishai/blob/efdcbe2b9265c5dac0c55b4ef2086e30a6121201/services/benchmark-runner/README.md",
  "qualificationImage": "us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca",
  "deployment": {
    "name": "bf-benchmark-api",
    "ready": 1,
    "image": [
      "us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:8ac7bb9c1c22f3ae1ee47e476242d4ce0ed71b7e0f3eb1dce6253d9fdee4cdfb"
    ]
  },
  "deploymentObservedAt": "2026-09-16T04:24:30.833051+00:00",
  "runActivity": [
    {
      "id": "br-7c3aab5279287890579fa7278dd0c911",
      "benchmark": "DevOpsBench-100",
      "status": "In progress",
      "observedAt": "2026-09-16T04:39:02.812754+00:00",
      "description": "Matched replacement run: ChatGPT, Claude and DeepSeek attempt the same ten original tasks after all three native world-tool controls passed. Scores are pending reviewed publication.",
      "plannedTasks": 10,
      "agents": 3
    },
    {
      "id": "br-68555897ccde9737d8d0f34d4dbf677e",
      "benchmark": "DevOpsBench-100",
      "status": "Cancelled \u00b7 harness qualification failure",
      "observedAt": "2026-09-16T03:36:18Z",
      "description": "The original tasks required 97 world tools, but the solver exposed only a terminal. The batch was cancelled, evidence retained, and the tool adapter repaired before the replacement run. This is a harness failure, not a company performance score.",
      "plannedTasks": 10,
      "agents": 3
    }
  ]
}