{"runs":[{"agent":{"id":"8dadcd93e81f5f5d6194ae7187575134b6cfe1d8c2e44fbc474d462c2ce45b86","model":"gpt-6-astra","name":"Codex + Playwright"},"benchmark":{"id":"online-mind2web","name":"Online-Mind2Web (GitHub snapshot pilot)","official_tasks":300,"version":"bab3fc3f1bff5b42c0624c073b68ddf9ad651d91"},"budget":"One attempt; 300 solver seconds; 60 browser actions; one task at a time; 7200s Job deadline","environment":"GKE bf-benchmarks; fresh gVisor Chromium per task; no model credentials in browser guest","evaluator":"Primary: rubric; declared grading contract 0ff5b01ba6f9e9a34b345cb2f62ec14e859ebbb6ad1f1188422c935356b544d3","expected_tasks":10,"finished_at":"2026-09-16T04:24:18.926543+00:00","id":"online-mind2web-codex-playwright-20260916:codex-playwright","notes":"Imported recorded evidence; not independently rerun by the report engine. Scaffold: harbor-playwright-dom-v1 at Harbor 0.21.0; Playwright 1.58.0; Codex CLI 0.154.0. Unknown measurements remain unknown. Fresh headless Chromium per task. Independent DOM rubric primary; at most eight uniformly spaced screenshots including first and last for supplemental vision. Native task-success grading is unconfigured. This is not the official WebJudge protocol or human adjudication. Solver and independent graders use the same model family; dollar costs are unmeasured. Task latency includes browser/solver setup and execution. Per-grader latency and total duration including grading are retained separately in raw outputs. Partial trajectories preserve original execution errors without rescoring.","primary_evaluator":"rubric","source":"imported","started_at":"2026-09-16T03:45:34.567280+00:00","tasks":[{"category":"www.bestbuy.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/515f2e5811cfdd5e0e669e40f17886d8-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/515f2e5811cfdd5e0e669e40f17886d8-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/515f2e5811cfdd5e0e669e40f17886d8.tar.gz"],"failure_mode":null,"id":"515f2e5811cfdd5e0e669e40f17886d8","latency_ms":335379.8730790004,"output":"","passed":null,"prompt":"Search for a new internal M2 Samsung SSD drive between $25 and $200.","reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.188:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"metadata":{},"model":"gpt-6-astra","passed":null,"reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.188:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","score":null,"status":"error","version":"browser-dom-constraints-v1"},"vision":{"metadata":{},"model":"gpt-6-astra","passed":null,"reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.188:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","score":null,"status":"error","version":"browser-uniform-eight-frames-v1"}},"status":"error","steps":15,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.fedex.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/6db4a0e346976f2729ba9afcd3208941-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/6db4a0e346976f2729ba9afcd3208941-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/6db4a0e346976f2729ba9afcd3208941.tar.gz"],"failure_mode":null,"id":"6db4a0e346976f2729ba9afcd3208941","latency_ms":43491.22251699737,"output":"I couldn’t retrieve tracking information for shipment **#3023858502**. The [FedEx website](https://www.fedex.com/en-us/home.html) blocked access, stating that I don’t have permission to view the page.","passed":false,"prompt":"Look up tracking information for shipment #3023858502.","reason":"The captured actions only opened the FedEx homepage and waited; neither used shipment number 3023858502. The page showed an access error, with no actual tracking information. The final answer cannot substitute for browser evidence.","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"latency_ms":23884.758401996805,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false],"engine":"codex","evidence_path":"codex-rubric.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-43d733d4ace8415683996059","pod_uid":"f3d3ee96-d7db-4415-8493-5747d2ac10bd","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":9431,"output_tokens":237,"reasoning_output_tokens":167}},"model":"gpt-6-astra","passed":false,"reason":"The captured actions only opened the FedEx homepage and waited; neither used shipment number 3023858502. The page showed an access error, with no actual tracking information. The final answer cannot substitute for browser evidence.","score":0.0,"status":"graded","version":"browser-dom-constraints-v1"},"vision":{"latency_ms":23427.780264999456,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false],"engine":"codex","evidence_path":"codex-vision.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-7830ed944aa444e797c84a58","pod_uid":"af0ecfaa-2ee8-4bb8-891d-965a864d2c5d","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":11971,"output_tokens":243,"reasoning_output_tokens":176}},"model":"gpt-6-astra","passed":false,"reason":"The actions and screenshots show only the FedEx homepage displaying an access error. They do not establish a tracking lookup for 3023858502 or provide actual shipment information, so the material requirements are unsupported.","score":0.0,"status":"graded","version":"browser-uniform-eight-frames-v1"}},"status":"completed","steps":2,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.microcenter.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/79f0bd7df6e685f30f20025cc6755c0a_110325-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/79f0bd7df6e685f30f20025cc6755c0a_110325-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/79f0bd7df6e685f30f20025cc6755c0a_110325.tar.gz"],"failure_mode":null,"id":"79f0bd7df6e685f30f20025cc6755c0a_110325","latency_ms":60383.54006300142,"output":"[Micro Center](https://www.microcenter.com/) remained blocked by Cloudflare’s “Performing security verification” page after waiting. I couldn’t access the products or produce a verified list of Mac-compatible external hard drives over 2TB sorted by price.","passed":false,"prompt":"Show me the list of external hard drives for Mac with a capacity greater than 2TB on Micro Center, sorted by price from lowest to highest.","reason":"Captured pages show only Cloudflare security verification. No external-drive list, explicit Mac compatibility, capacity filtering strictly above 2 TB, or lowest-to-highest price sorting is supported by the browser evidence.","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"latency_ms":24500.989645999653,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false,false],"engine":"codex","evidence_path":"codex-rubric.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-2474658d30ce4230b6479093","pod_uid":"a69d2e66-4168-43e7-bf9f-21505e347690","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":6912,"input_tokens":10125,"output_tokens":214,"reasoning_output_tokens":147}},"model":"gpt-6-astra","passed":false,"reason":"Captured pages show only Cloudflare security verification. No external-drive list, explicit Mac compatibility, capacity filtering strictly above 2 TB, or lowest-to-highest price sorting is supported by the browser evidence.","score":0.0,"status":"graded","version":"browser-dom-constraints-v1"},"vision":{"latency_ms":28465.94424199793,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false,false],"engine":"codex","evidence_path":"codex-vision.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-eb34a16718374792ba9a79bd","pod_uid":"4c5f634a-f3e5-47d7-b2c2-538578fa849a","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":14935,"output_tokens":201,"reasoning_output_tokens":135}},"model":"gpt-6-astra","passed":false,"reason":"The screenshots show only Cloudflare verification, and the browser actions only navigate and wait. No Mac-compatible external drive list, capacity filtering strictly above 2 TB, or ascending price sorting is demonstrated.","score":0.0,"status":"graded","version":"browser-uniform-eight-frames-v1"}},"status":"completed","steps":4,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.theweathernetwork.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/871e7771cecb989972f138ecc373107b-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/871e7771cecb989972f138ecc373107b-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/871e7771cecb989972f138ecc373107b.tar.gz"],"failure_mode":null,"id":"871e7771cecb989972f138ecc373107b","latency_ms":115192.63836000027,"output":"Vancouver, British Columbia’s seven-day forecast for **September 16–22, 2026**, from [The Weather Network](https://www.theweathernetwork.com/en/city/ca/british-columbia/vancouver/7-days):\n\n| Day | Forecast | Afternoon / Overnight |\n|---|---|---|\n| Wed, Sep 16 | Sunny; clear overnight | 21°C / 12°C |\n| Thu, Sep 17 | Sunny; fog patches overnight | 21°C / 12°C |\n| Fri, Sep 18 | Morning fog, then sunny | 21°C / 12°C |\n| Sat, Sep 19 | Sun and clouds, becoming mainly sunny | 20°C / 12°C |\n| Sun, Sep 20 | Mainly sunny | 22°C / 12°C |\n| Mon, Sep 21 | Sun and clouds; chance of showers overnight | 20°C / 13°C |\n| Tue, Sep 22 | Showers possible; light rain overnight | 17°C / 10°C |\n\nMostly dry through the weekend, with rain chances rising to **40–60%** Monday night into Tuesday.","passed":true,"prompt":"Find the weather for Vancouver, British Columbia for the next seven days.","reason":"Browser actions and captured text confirm The Weather Network’s Vancouver, British Columbia page. Step 4 supports temperatures and conditions for all seven days, September 16–22, 2026, matching the run date and reported forecast.","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"latency_ms":32296.372149998206,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[true,true,true],"engine":"codex","evidence_path":"codex-rubric.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-0aa4a8cade074238a9141379","pod_uid":"d551aaf1-b1aa-4318-bdd8-c52b889a0400","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":37467,"output_tokens":400,"reasoning_output_tokens":328}},"model":"gpt-6-astra","passed":true,"reason":"Browser actions and captured text confirm The Weather Network’s Vancouver, British Columbia page. Step 4 supports temperatures and conditions for all seven days, September 16–22, 2026, matching the run date and reported forecast.","score":1.0,"status":"graded","version":"browser-dom-constraints-v1"},"vision":{"latency_ms":27893.841302997316,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[true,false,false],"engine":"codex","evidence_path":"codex-vision.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-22f2030cb8f9491ba64a5f54","pod_uid":"ba75942f-c482-4fba-b5dc-355c272a5b77","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":16554,"output_tokens":412,"reasoning_output_tokens":336}},"model":"gpt-6-astra","passed":false,"reason":"The Weather Network page confirms Vancouver, British Columbia. However, screenshots show only September 15 overnight and portions of September 16–18, not the full September 16–22 forecast. The remaining days reported in the final answer lack captured evidence.","score":0.3333333333333333,"status":"graded","version":"browser-uniform-eight-frames-v1"}},"status":"completed","steps":5,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.bestbuy.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/87f4c5128e36cdb9366a138a7b61bb00-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/87f4c5128e36cdb9366a138a7b61bb00-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/87f4c5128e36cdb9366a138a7b61bb00.tar.gz"],"failure_mode":null,"id":"87f4c5128e36cdb9366a138a7b61bb00","latency_ms":334044.578762001,"output":"","passed":null,"prompt":"View the speakers that are bluetooth and wireless and filter the results to only show models that are on sale and cost less than $50.","reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.145:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"metadata":{},"model":"gpt-6-astra","passed":null,"reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.145:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","score":null,"status":"error","version":"browser-dom-constraints-v1"},"vision":{"metadata":{},"model":"gpt-6-astra","passed":null,"reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.145:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","score":null,"status":"error","version":"browser-uniform-eight-frames-v1"}},"status":"error","steps":11,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.target.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/b64f938af842f6a1b4489d0e49a785a7_121125-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/b64f938af842f6a1b4489d0e49a785a7_121125-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/b64f938af842f6a1b4489d0e49a785a7_121125.tar.gz"],"failure_mode":null,"id":"b64f938af842f6a1b4489d0e49a785a7_121125","latency_ms":319214.0596169993,"output":"Found [Yough! Mozzarella Cheese Frozen Pizza, 11.3 oz](https://www.target.com/p/yough-mozzarella-cheese-frozen-pizza-11-3oz/-/A-94975371) for **$9.99**, listed under Target’s vegetarian filter.\n\nAvailable for pickup at Council Bluffs. A CAPTCHA blocked further interaction, so it wasn’t added to the cart.","passed":true,"prompt":"Get the frozen vegetarian cheese pizza between 5 to 10 USD on Target.","reason":"Captured results show Yough! Mozzarella Cheese Frozen Pizza under the active Vegetarian filter. Its Target product page confirms the frozen cheese pizza and $9.99 price, satisfying all requirements independently of the final answer.","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"latency_ms":25339.617286001157,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[true,true,true],"engine":"codex","evidence_path":"codex-rubric.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-3bfefa531789414a816dd818","pod_uid":"8d18493f-72ca-45fc-8ebf-cb7ff7e71cf7","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":87063,"output_tokens":337,"reasoning_output_tokens":268}},"model":"gpt-6-astra","passed":true,"reason":"Captured results show Yough! Mozzarella Cheese Frozen Pizza under the active Vegetarian filter. Its Target product page confirms the frozen cheese pizza and $9.99 price, satisfying all requirements independently of the final answer.","score":1.0,"status":"graded","version":"browser-dom-constraints-v1"},"vision":{"latency_ms":28387.68698000058,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false],"engine":"codex","evidence_path":"codex-vision.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-ef99e56ef2e846e78d50845b","pod_uid":"106fd335-06f6-46ad-8321-3029ea3e289e","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":6912,"input_tokens":21182,"output_tokens":337,"reasoning_output_tokens":266}},"model":"gpt-6-astra","passed":false,"reason":"The screenshots and browser actions do not verify a product’s vegetarian status or show a price between $5 and $10. The claimed product and $9.99 price appear only in the final answer, which cannot establish success.","score":0.0,"status":"graded","version":"browser-uniform-eight-frames-v1"}},"status":"completed","steps":12,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"jobs.chronicle.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/b922508886ded315c9835457a6eb43ea-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/b922508886ded315c9835457a6eb43ea-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/b922508886ded315c9835457a6eb43ea.tar.gz"],"failure_mode":null,"id":"b922508886ded315c9835457a6eb43ea","latency_ms":152759.4937879985,"output":"Found **2 listings** under Computer Sciences & Technology → Tenured/Tenured Track → California:\n\n- [Claremont Colleges Joint Faculty Positions](https://jobs.chronicle.com/job/38032680/claremont-colleges-joint-faculty-positions/) — includes Pomona College’s tenure-track Assistant Professor of Computer Science, open specialization.\n- [UC San Diego: Director, Halicioglu Data Science Institute](https://jobs.chronicle.com/job/38017541/director-halicioglu-data-science-institute) — listed salary **$200,000–$400,000**; categorized by the site as tenured/tenure-track.\n\nThe browser is on the [filtered results page](https://jobs.chronicle.com/jobs/computer-sciences-and-technology/tenured-tenured-track/california/).","passed":true,"prompt":"Browse tenured/tenure-track faculty positions in Computer Sciences & Technology in California.","reason":"Browser actions applied the Computer Sciences & Technology, Tenured/Tenured Track, and California filters on Chronicle Jobs. Captured page text at steps 5 and 9 confirms all filters and displays two matching listings.","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"latency_ms":26396.420527002192,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[true,true,true,true],"engine":"codex","evidence_path":"codex-rubric.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-02a28caec1b34e29b1159ec2","pod_uid":"a35a3ca1-6c28-4557-ba1a-cf0f5ce321b7","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":61412,"output_tokens":318,"reasoning_output_tokens":248}},"model":"gpt-6-astra","passed":true,"reason":"Browser actions applied the Computer Sciences & Technology, Tenured/Tenured Track, and California filters on Chronicle Jobs. Captured page text at steps 5 and 9 confirms all filters and displays two matching listings.","score":1.0,"status":"graded","version":"browser-dom-constraints-v1"},"vision":{"latency_ms":36465.890414998285,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[true,true,true,true],"engine":"codex","evidence_path":"codex-vision.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-7c59ef7d1c2f4546ac3f4804","pod_uid":"7b12061a-8569-44a2-bf81-2ef15892e40e","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":21257,"output_tokens":582,"reasoning_output_tokens":516}},"model":"gpt-6-astra","passed":true,"reason":"Screenshots and successful browser actions confirm Chronicle’s Computer Sciences & Technology faculty category with Tenured/Tenured Track and California filters applied. The filtered results page displays “Found 2 jobs.”","score":1.0,"status":"graded","version":"browser-uniform-eight-frames-v1"}},"status":"completed","steps":10,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.ikea.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/c6c9dc6079677cef594cec2fa6b16602-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/c6c9dc6079677cef594cec2fa6b16602-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/c6c9dc6079677cef594cec2fa6b16602.tar.gz"],"failure_mode":null,"id":"c6c9dc6079677cef594cec2fa6b16602","latency_ms":314159.6116119981,"output":"","passed":null,"prompt":"Add the cheapest black sofa with at least three seats, a leather finish, and at least four stars to my cart.","reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.155:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"metadata":{},"model":"gpt-6-astra","passed":null,"reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.155:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","score":null,"status":"error","version":"browser-dom-constraints-v1"},"vision":{"metadata":{},"model":"gpt-6-astra","passed":null,"reason":"RuntimeError: Server error '502 Bad Gateway' for url 'http://10.40.192.155:8765/solve'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502","score":null,"status":"error","version":"browser-uniform-eight-frames-v1"}},"status":"error","steps":20,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.adoptapet.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/cad62d2be0c53f08a416457486b3db23-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/cad62d2be0c53f08a416457486b3db23-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/cad62d2be0c53f08a416457486b3db23.tar.gz"],"failure_mode":null,"id":"cad62d2be0c53f08a416457486b3db23","latency_ms":42387.29839100051,"output":"I couldn’t search for adoptable dogs near 21122 because [Adopt a Pet](https://www.adoptapet.com/) returned a **403 “Request blocked”** error. No dog listings were accessible.","passed":false,"prompt":"Search for adoptable dogs near 21122 zip code.","reason":"The evidence shows only navigation to Adopt a Pet and a 403 error. No dog search using ZIP code 21122, local matches, or confirmed empty result is captured.","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"latency_ms":27565.145582000696,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false],"engine":"codex","evidence_path":"codex-rubric.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-3414d2d952a34d7bbe5b8373","pod_uid":"df9f24c7-2d15-413e-bf43-437cb89a3a1e","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":0,"input_tokens":9540,"output_tokens":247,"reasoning_output_tokens":186}},"model":"gpt-6-astra","passed":false,"reason":"The evidence shows only navigation to Adopt a Pet and a 403 error. No dog search using ZIP code 21122, local matches, or confirmed empty result is captured.","score":0.0,"status":"graded","version":"browser-dom-constraints-v1"},"vision":{"latency_ms":30400.866880998365,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false],"engine":"codex","evidence_path":"codex-vision.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-07183bd96d46488c9694bff6","pod_uid":"f4f71de9-42bd-4afe-8aab-32ced426f2e5","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":6912,"input_tokens":11864,"output_tokens":208,"reasoning_output_tokens":138}},"model":"gpt-6-astra","passed":false,"reason":"The screenshots show only a 403 error, and browser actions show navigation to Adopt a Pet and waiting. There is no evidence of a dog search using ZIP 21122, local matches, or a confirmed empty result.","score":0.0,"status":"graded","version":"browser-uniform-eight-frames-v1"}},"status":"completed","steps":2,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."},{"category":"www.google.com","cost_usd":null,"evidence":["https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/e24662008c3be5d56f986f232fcec447-final.png","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/e24662008c3be5d56f986f232fcec447-trajectory.json","https://blobfish.ai/agent-reports/evidence/online-mind2web-codex-playwright-20260916/e24662008c3be5d56f986f232fcec447.tar.gz"],"failure_mode":null,"id":"e24662008c3be5d56f986f232fcec447","latency_ms":112195.69871400017,"output":"Google Finance lists WWE under its parent, **TKO Group Holdings (NYSE: TKO)**.\n\n- **Latest close:** $190.13 USD on September 15, 2026.\n- **Last month:** Down **$5.01 (2.57%)**.\n\n[View the one-month stock chart](https://www.google.com/finance/beta/quote/TKO:NYSE?window=1M).","passed":false,"prompt":"Find the stock price for WWE over the last month.","reason":"The captured Google Finance page shows TKO:NYSE with a one-month chart. This substitutes WWE’s parent company, explicitly prohibited by the criteria. No captured page text or browser action establishes WWE’s own stock-price history for the requested recent month.","scoring":{"native":{"metadata":{},"model":null,"passed":null,"reason":"Evaluator was not configured","score":null,"status":"not_configured","version":"not-configured"},"rubric":{"latency_ms":33332.84589100003,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false],"engine":"codex","evidence_path":"codex-rubric.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-326b65e95dd24c59a7639679","pod_uid":"77e71d25-fb66-41c4-886a-42896fb572fe","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":6912,"input_tokens":52677,"output_tokens":424,"reasoning_output_tokens":349}},"model":"gpt-6-astra","passed":false,"reason":"The captured Google Finance page shows TKO:NYSE with a one-month chart. This substitutes WWE’s parent company, explicitly prohibited by the criteria. No captured page text or browser action establishes WWE’s own stock-price history for the requested recent month.","score":0.0,"status":"graded","version":"browser-dom-constraints-v1"},"vision":{"latency_ms":26285.37358200265,"metadata":{"account":"eval-01","auth":"chatgpt_subscription","cli_version":"0.154.0","criteria_met":[false,false,false],"engine":"codex","evidence_path":"codex-vision.json","image":"us-central1-docker.pkg.dev/blobfish-ai-429200/cloud-run-source-deploy/bf-benchmark-runner@sha256:c29c39cc466ea083014a702664844a2304a67c1a01a31f29d14233d6ef9a97ca","pod_name":"bt-7715ceed53054e7d8104383f","pod_uid":"a5058a73-6139-494d-94cb-d90112d40b05","reasoning_effort":"max","usage":{"cache_write_input_tokens":0,"cached_input_tokens":6912,"input_tokens":20910,"output_tokens":396,"reasoning_output_tokens":328}},"model":"gpt-6-astra","passed":false,"reason":"The screenshots and browser actions show a recent one-month chart for TKO Group Holdings, not WWE. Substituting its parent company is explicitly prohibited, and WWE’s requested recent stock-price history is not demonstrated.","score":0.0,"status":"graded","version":"browser-uniform-eight-frames-v1"}},"status":"completed","steps":8,"trace":"Full recorded inputs, outputs and grading metadata are in raw_outputs.jsonl."}]}],"schema_version":1,"title":"Online-Mind2Web (GitHub snapshot pilot) · agent evaluation"}