{"export":{"agents":[{"id":"codex-playwright","model":"gpt-6-astra","name":"Codex + Playwright","scaffold":"harbor-playwright-dom-v1","version":"Harbor 0.21.0; Playwright 1.58.0; Codex CLI 0.154.0"}],"benchmark":{"id":"online-mind2web","name":"Online-Mind2Web (GitHub snapshot pilot)","official_tasks":300,"version":"bab3fc3f1bff5b42c0624c073b68ddf9ad651d91"},"budget":"One attempt; 300 solver seconds; 60 browser actions; one task at a time; 7200s Job deadline","environment":"GKE bf-benchmarks; fresh gVisor Chromium per task; no model credentials in browser guest","finished_at":"2026-09-16T04:24:18.926543+00:00","graders":{"rubric":{"model":"gpt-6-astra","version":"browser-dom-constraints-v1"},"vision":{"model":"gpt-6-astra","version":"browser-uniform-eight-frames-v1"}},"id":"online-mind2web-codex-playwright-20260916","notes":"Fresh headless Chromium per task. Independent DOM rubric primary; at most eight uniformly spaced screenshots including first and last for supplemental vision. Native task-success grading is unconfigured. This is not the official WebJudge protocol or human adjudication. Solver and independent graders use the same model family; dollar costs are unmeasured.","primary_evaluator":"rubric","started_at":"2026-09-16T03:45:34.567280+00:00","tasks":[{"id":"515f2e5811cfdd5e0e669e40f17886d8","prompt":"Search for a new internal M2 Samsung SSD drive between $25 and $200.","site":"www.bestbuy.com","url":"https://www.bestbuy.com/"},{"id":"6db4a0e346976f2729ba9afcd3208941","prompt":"Look up tracking information for shipment #3023858502.","site":"www.fedex.com","url":"https://www.fedex.com/en-us/home.html"},{"id":"79f0bd7df6e685f30f20025cc6755c0a_110325","prompt":"Show me the list of external hard drives for Mac with a capacity greater than 2TB on Micro Center, sorted by price from lowest to highest.","site":"www.microcenter.com","url":"https://www.microcenter.com/"},{"id":"871e7771cecb989972f138ecc373107b","prompt":"Find the weather for Vancouver, British Columbia for the next seven days.","site":"www.theweathernetwork.com","url":"https://www.theweathernetwork.com/"},{"id":"87f4c5128e36cdb9366a138a7b61bb00","prompt":"View the speakers that are bluetooth and wireless and filter the results to only show models that are on sale and cost less than $50.","site":"www.bestbuy.com","url":"https://www.bestbuy.com/"},{"id":"b64f938af842f6a1b4489d0e49a785a7_121125","prompt":"Get the frozen vegetarian cheese pizza between 5 to 10 USD on Target.","site":"www.target.com","url":"https://www.target.com/"},{"id":"b922508886ded315c9835457a6eb43ea","prompt":"Browse tenured/tenure-track faculty positions in Computer Sciences & Technology in California.","site":"jobs.chronicle.com","url":"https://jobs.chronicle.com"},{"id":"c6c9dc6079677cef594cec2fa6b16602","prompt":"Add the cheapest black sofa with at least three seats, a leather finish, and at least four stars to my cart.","site":"www.ikea.com","url":"https://www.ikea.com/"},{"id":"cad62d2be0c53f08a416457486b3db23","prompt":"Search for adoptable dogs near 21122 zip code.","site":"www.adoptapet.com","url":"https://www.adoptapet.com/"},{"id":"e24662008c3be5d56f986f232fcec447","prompt":"Find the stock price for WWE over the last month.","site":"www.google.com","url":"https://www.google.com/finance/"}]},"input":{"agent":{"id":"codex-playwright","max_steps":60,"model":"gpt-6-astra","name":"Codex + Playwright","provider":"openai","scaffold":"harbor-codex-subscription"},"benchmark":"online-mind2web","dataset_lock":{"adapter_version":"blobfish-benchmark-datasets/0.1.0","available_rows":300,"files":[{"bytes":88329,"path":"benchmarks/jsons/online_mind2web.json","sha256":"50d330ee9f347c5ae136e52421516590033b146df305ab091c5a36b199b14aa4"}],"mapping":{"id":"task_id","prompt":"confirmed_task","url":"website"},"profile":"online-mind2web","qualification":"Prepared dataset, not an executed or graded benchmark result.","schema_version":1,"selected_rows":10,"selection":{"algorithm":"sha256-seed-source-id-v1","seed":"20260916","task_ids":["515f2e5811cfdd5e0e669e40f17886d8","6db4a0e346976f2729ba9afcd3208941","79f0bd7df6e685f30f20025cc6755c0a_110325","871e7771cecb989972f138ecc373107b","87f4c5128e36cdb9366a138a7b61bb00","b64f938af842f6a1b4489d0e49a785a7_121125","b922508886ded315c9835457a6eb43ea","c6c9dc6079677cef594cec2fa6b16602","cad62d2be0c53f08a416457486b3db23","e24662008c3be5d56f986f232fcec447"]},"source":{"files":["benchmarks/jsons/online_mind2web.json"],"kind":"github","repo":"allenai/molmoweb","revision":"bab3fc3f1bff5b42c0624c073b68ddf9ad651d91"},"source_metadata":{"repository":"allenai/molmoweb","revision":"bab3fc3f1bff5b42c0624c073b68ddf9ad651d91"},"tasks_sha256":"1bbbf312a9480a6cff530567b6645fdb8a1047abd2ac1faeb0df6a78a7373107"},"dataset_lock_sha256":"d592ad068cb72c19648ee48dd8cff86ce740936144881623c6f8191f8e33ed9d","id":"online-mind2web-codex-playwright-20260916","max_steps":60,"primary_evaluator":"rubric","protocol":"Fresh headless Chromium per task. Independent DOM rubric primary; at most eight uniformly spaced screenshots including first and last for supplemental vision. Native task-success grading is unconfigured. This is not the official WebJudge protocol or human adjudication.","scaffold":"harbor-playwright-dom-v1","schema_version":1,"task_timeout_sec":300,"tasks":[{"graders":{"rubric":{"criteria":["Find a new internal M.2 Samsung SSD on Best Buy.","The observed item or filtered results must be priced between USD 25 and USD 200 inclusive.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Find a new internal M.2 Samsung SSD on Best Buy.","The observed item or filtered results must be priced between USD 25 and USD 200 inclusive.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"515f2e5811cfdd5e0e669e40f17886d8","prompt":"Search for a new internal M2 Samsung SSD drive between $25 and $200.","reference_length":8,"site":"www.bestbuy.com","source_row":{"confirmed_task":"Search for a new internal M2 Samsung SSD drive between $25 and $200.","level":"medium","reference_length":8,"task_id":"515f2e5811cfdd5e0e669e40f17886d8","website":"https://www.bestbuy.com/"},"url":"https://www.bestbuy.com/"},{"graders":{"rubric":{"criteria":["Use FedEx tracking for the exact shipment number 3023858502.","Return actual shipment tracking information for that number. A missing/invalid number message or an access error does not fulfill the request; do not infer delivery details.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Use FedEx tracking for the exact shipment number 3023858502.","Return actual shipment tracking information for that number. A missing/invalid number message or an access error does not fulfill the request; do not infer delivery details.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"6db4a0e346976f2729ba9afcd3208941","prompt":"Look up tracking information for shipment #3023858502.","reference_length":2,"site":"www.fedex.com","source_row":{"confirmed_task":"Look up tracking information for shipment #3023858502.","level":"easy","reference_length":2,"task_id":"6db4a0e346976f2729ba9afcd3208941","website":"https://www.fedex.com/en-us/home.html"},"url":"https://www.fedex.com/en-us/home.html"},{"graders":{"rubric":{"criteria":["Show a Micro Center list of external hard drives explicitly compatible with Mac.","All requested capacity filtering must be greater than 2 TB, not 2 TB or greater.","The resulting list must be sorted by price from lowest to highest.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Show a Micro Center list of external hard drives explicitly compatible with Mac.","All requested capacity filtering must be greater than 2 TB, not 2 TB or greater.","The resulting list must be sorted by price from lowest to highest.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"79f0bd7df6e685f30f20025cc6755c0a_110325","prompt":"Show me the list of external hard drives for Mac with a capacity greater than 2TB on Micro Center, sorted by price from lowest to highest.","reference_length":12,"site":"www.microcenter.com","source_row":{"confirmed_task":"Show me the list of external hard drives for Mac with a capacity greater than 2TB on Micro Center, sorted by price from lowest to highest.","level":"hard","reference_length":12,"task_id":"79f0bd7df6e685f30f20025cc6755c0a_110325","website":"https://www.microcenter.com/"},"url":"https://www.microcenter.com/"},{"graders":{"rubric":{"criteria":["Use The Weather Network for Vancouver in British Columbia, Canada, not Vancouver in Washington.","Show or report the forecast for the next seven days relative to the actual run date; all seven days must be supported by captured evidence.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Use The Weather Network for Vancouver in British Columbia, Canada, not Vancouver in Washington.","Show or report the forecast for the next seven days relative to the actual run date; all seven days must be supported by captured evidence.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"871e7771cecb989972f138ecc373107b","prompt":"Find the weather for Vancouver, British Columbia for the next seven days.","reference_length":3,"site":"www.theweathernetwork.com","source_row":{"confirmed_task":"Find the weather for Vancouver, British Columbia for the next seven days.","level":"easy","reference_length":3,"task_id":"871e7771cecb989972f138ecc373107b","website":"https://www.theweathernetwork.com/"},"url":"https://www.theweathernetwork.com/"},{"graders":{"rubric":{"criteria":["On Best Buy display speaker results that are both Bluetooth and wireless.","Filter the results to models that are on sale and cost strictly less than USD 50.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["On Best Buy display speaker results that are both Bluetooth and wireless.","Filter the results to models that are on sale and cost strictly less than USD 50.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"87f4c5128e36cdb9366a138a7b61bb00","prompt":"View the speakers that are bluetooth and wireless and filter the results to only show models that are on sale and cost less than $50.","reference_length":6,"site":"www.bestbuy.com","source_row":{"confirmed_task":"View the speakers that are bluetooth and wireless and filter the results to only show models that are on sale and cost less than $50.","level":"medium","reference_length":6,"task_id":"87f4c5128e36cdb9366a138a7b61bb00","website":"https://www.bestbuy.com/"},"url":"https://www.bestbuy.com/"},{"graders":{"rubric":{"criteria":["Find a product on Target that is explicitly a frozen vegetarian cheese pizza.","The observed product price must be between USD 5 and USD 10 inclusive.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Find a product on Target that is explicitly a frozen vegetarian cheese pizza.","The observed product price must be between USD 5 and USD 10 inclusive.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"b64f938af842f6a1b4489d0e49a785a7_121125","prompt":"Get the frozen vegetarian cheese pizza between 5 to 10 USD on Target.","reference_length":17,"site":"www.target.com","source_row":{"confirmed_task":"Get the frozen vegetarian cheese pizza between 5 to 10 USD on Target.","level":"hard","reference_length":17,"task_id":"b64f938af842f6a1b4489d0e49a785a7_121125","website":"https://www.target.com/"},"url":"https://www.target.com/"},{"graders":{"rubric":{"criteria":["Browse jobs on The Chronicle of Higher Education in the Computer Sciences & Technology category.","Restrict positions to tenured or tenure-track faculty in California.","Display matching results or a confirmed empty result after those exact filters are applied.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Browse jobs on The Chronicle of Higher Education in the Computer Sciences & Technology category.","Restrict positions to tenured or tenure-track faculty in California.","Display matching results or a confirmed empty result after those exact filters are applied.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"b922508886ded315c9835457a6eb43ea","prompt":"Browse tenured/tenure-track faculty positions in Computer Sciences & Technology in California.","reference_length":6,"site":"jobs.chronicle.com","source_row":{"confirmed_task":"Browse tenured/tenure-track faculty positions in Computer Sciences & Technology in California.","level":"medium","reference_length":6,"task_id":"b922508886ded315c9835457a6eb43ea","website":"https://jobs.chronicle.com"},"url":"https://jobs.chronicle.com"},{"graders":{"rubric":{"criteria":["On IKEA find a black sofa with at least three seats, a leather finish, and a customer rating of at least four stars.","Establish that the chosen sofa is the cheapest among products meeting all those constraints; a single product price alone is insufficient.","Add that qualifying sofa to the anonymous cart and observe confirmation. No purchase is required.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["On IKEA find a black sofa with at least three seats, a leather finish, and a customer rating of at least four stars.","Establish that the chosen sofa is the cheapest among products meeting all those constraints; a single product price alone is insufficient.","Add that qualifying sofa to the anonymous cart and observe confirmation. No purchase is required.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"c6c9dc6079677cef594cec2fa6b16602","prompt":"Add the cheapest black sofa with at least three seats, a leather finish, and at least four stars to my cart.","reference_length":16,"site":"www.ikea.com","source_row":{"confirmed_task":"Add the cheapest black sofa with at least three seats, a leather finish, and at least four stars to my cart.","level":"hard","reference_length":16,"task_id":"c6c9dc6079677cef594cec2fa6b16602","website":"https://www.ikea.com/"},"url":"https://www.ikea.com/"},{"graders":{"rubric":{"criteria":["Use Adopt a Pet to search for adoptable dogs, not another animal type.","The search must use ZIP code 21122 and show the resulting local matches or a confirmed empty result for those exact filters.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Use Adopt a Pet to search for adoptable dogs, not another animal type.","The search must use ZIP code 21122 and show the resulting local matches or a confirmed empty result for those exact filters.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"cad62d2be0c53f08a416457486b3db23","prompt":"Search for adoptable dogs near 21122 zip code.","reference_length":5,"site":"www.adoptapet.com","source_row":{"confirmed_task":"Search for adoptable dogs near 21122 zip code.","level":"easy","reference_length":5,"task_id":"cad62d2be0c53f08a416457486b3db23","website":"https://www.adoptapet.com/"},"url":"https://www.adoptapet.com/"},{"graders":{"rubric":{"criteria":["Use Google Finance for WWE's stock-price history over the last month relative to the actual run date.","Do not substitute a different company or an old date range. If WWE is unavailable or delisted and the requested recent history cannot be observed, the task is not fulfilled, even when the limitation is correctly explained.","Every material requirement above must be supported by the captured page text and factual browser actions. The final answer alone is not evidence."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-dom-constraints-v1"},"vision":{"criteria":["Use Google Finance for WWE's stock-price history over the last month relative to the actual run date.","Do not substitute a different company or an old date range. If WWE is unavailable or delisted and the requested recent history cannot be observed, the task is not fulfilled, even when the limitation is correctly explained.","Every material requirement above must be supported by the supplied screenshots and factual browser actions. The final answer alone is not evidence; unseen frames cannot establish success."],"engine":"codex","model":"gpt-6-astra","reference":"","screenshots":[],"version":"browser-uniform-eight-frames-v1"}},"id":"e24662008c3be5d56f986f232fcec447","prompt":"Find the stock price for WWE over the last month.","reference_length":4,"site":"www.google.com","source_row":{"confirmed_task":"Find the stock price for WWE over the last month.","level":"easy","reference_length":4,"task_id":"e24662008c3be5d56f986f232fcec447","website":"https://www.google.com/finance/"},"url":"https://www.google.com/finance/"}]},"submission":{"browser_image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-browser-benchmark@sha256:0f40f167dc21b05ead665c6756f01adb36cdee2a75a079cc354878b80e09b771","bucket":"blobfish-benchmark-runs-blobfish-ai-429200","configmap_uid":"8a04c0ed-1de0-4a90-9d6b-ffb2cdf3f6e8","input_sha256":"011f414937375b00451e462bdf4201964bf161b35eee7be05947f5be644fb994","job_uid":"514cb6a1-e125-4339-b24c-c40280d111a2","name":"bw-d640d435d111ce45df7bcbe3","namespace":"bf-benchmarks","prefix":"browser-runs/online-mind2web-codex-playwright-20260916/d640d435d111ce45df7bcbe3836b5a84fdca17a2697ed0ea7756a919e1f0541c","runner_image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","scripts_sha256":{"browser.py":"f3f9d0c8bd7b1e9b1ffa3e6f25e206806e8e050e8e19ebe2742f90d586e55d41","browser_benchmark_gke.py":"f1562ad8f1b07a9640c2543ae481eabbc5416daafbb391afc6c2c443644a7156"},"source_commit":"d501fec27f9bfcb41fc8359bd08c50c645a4d737","submitted_at":"2026-09-16T03:45:26.805586+00:00"}}