{ "schemaVersion": "blobfish.typed-decision-bench-task.v1", "version": "0.3.0", "task": "tau2-airline-run-success", "family": "real-time-and-agents", "use_case": "run-judging", "vertical": "customer-operations", "dataset": "snorkelai/Tau2-Bench-Verified-Airline-With-Code-Agents", "licence": "Apache-2.0", "provenance": "verifier", "state_kind": "trajectory", "primitive": "noul", "items": 200, "scores": { "decider-2b": { "decisionScore": 67.32, "accuracy": 0.47, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 68.02, "accuracy": 0.52, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 54.88, "accuracy": 0.47, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 75.47, "accuracy": 0.5, "answered": 198, "items": 200 }, "OpenJev": { "decisionScore": 55.26, "accuracy": 0.475, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 71.85, "accuracy": 0.485, "answered": 200, "items": 200 } }, "samples": [ { "id": "real-time-and-agents:tau2-0b7422246c73", "state": { "domain": "airline", "customer_goal": "For your upcoming trip from New York to Chicago, you want to change the passenger to yourself, upgrade it to economy class, and have 3 checked bags.", "db_update_required": true, "n_turns": 38, "tool_sequence": [ "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "get_datetime", "code_evaluation" ], "last_actions": [ { "name": "get_datetime", "args": "{}" }, { "name": "code_evaluation", "args": "{\"expression\": \"# Let's process the reservation changes\\nfrom tau2.domains.airline_open.data_model import Passenger, …" } ], "last_user_message": "No, that’s all I needed. Thanks for your help! ###STOP###", "final_agent_message": "Excellent! Your reservation changes have been successfully processed. Here's a summary of what was completed:\n\n## ✅ Reservation FQ8APE - Changes Complete\n\n**Updated Details:**\n- **Passenger**: Omar Rossi (you)\n- **Cabin**: Economy (upgraded from Basic Economy)\n- **Baggage**: 3 checked bags total …" }, "state_tokens": 243, "tier": "common", "primary": "success", "questions": [ { "id": "success", "type": "noul", "instructions": "Given `customer_goal`, `db_update_required`, `tool_sequence`, `last_actions`, `last_user_message` and `final_agent_message`, did the agent end the session with the airline database in the correct state and with the required information communicated to the customer, according to airline policy?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.9241, "no": 0.0759 }, "probability_of_expected": 0.0759, "proper_score": 0.146 }, "Jev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.5, "no": 0.5 }, "probability_of_expected": 0.5, "proper_score": 0.75 }, "JevFish": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7133, "no": 0.2867 }, "probability_of_expected": 0.2867, "proper_score": 0.4912 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7607, "no": 0.2393 }, "probability_of_expected": 0.2393, "proper_score": 0.4214 }, "OpenJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 1.0, "no": 0.0 }, "probability_of_expected": 0.0, "proper_score": 0.0001 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.8199, "no": 0.1801 }, "probability_of_expected": 0.1801, "proper_score": 0.3277, "truncated": true } } }, { "id": "db_correct", "type": "noul", "instructions": "Did the session end with the airline database in exactly the state the customer's request and airline policy require (no missing and no extra changes)?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.9095, "no": 0.0905 }, "probability_of_expected": 0.0905, "proper_score": 0.1728 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.44, "no": 0.56 }, "probability_of_expected": 0.56, "proper_score": 0.8064 }, "JevFish": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7274, "no": 0.2726 }, "probability_of_expected": 0.2726, "proper_score": 0.4709 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6451, "no": 0.3549 }, "probability_of_expected": 0.3549, "proper_score": 0.5838 }, "OpenJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 1.0, "no": 0.0 }, "probability_of_expected": 0.0, "proper_score": 0.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7925, "no": 0.2075 }, "probability_of_expected": 0.2075, "proper_score": 0.372 } } }, { "id": "communicated", "type": "noul", "instructions": "Did the agent tell the customer every piece of information the request required?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9559, "no": 0.0441 }, "probability_of_expected": 0.9559, "proper_score": 0.9981 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.7, "no": 0.3 }, "probability_of_expected": 0.7, "proper_score": 0.91 }, "JevFish": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.585, "no": 0.415 }, "probability_of_expected": 0.585, "proper_score": 0.8278 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.6298, "no": 0.3702 }, "probability_of_expected": 0.6298, "proper_score": 0.8629 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 1.0, "no": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.6466, "no": 0.3534 }, "probability_of_expected": 0.6466, "proper_score": 0.8751 } } } ], "latency_ms": { "decider-2b": 7398.9, "Jev": 192.9, "JevFish": 3084.8, "NanoJev": 741.6, "OpenJev": 343.2, "System One Scorer": 3300.8 } }, { "id": "real-time-and-agents:tau2-394ebb4d5efa", "state": { "domain": "airline", "customer_goal": "You want to cancel all of your upcoming flights.", "db_update_required": true, "n_turns": 30, "tool_sequence": [ "get_user_details", "get_reservation_details", "get_reservation_details", "get_reservation_details", "get_reservation_details", "get_reservation_details", "get_reservation_details", "get_reservation_details", "cancel_reservation", "cancel_reservation" ], "last_actions": [ { "name": "cancel_reservation", "args": "{\"reservation_id\": \"8C8K4E\"}" }, { "name": "cancel_reservation", "args": "{\"reservation_id\": \"LU15PA\"}" } ], "last_user_message": "Merci beaucoup pour votre aide et votre gentillesse. Je comprends pour les autres réservations, ce n’est pas grave. Bonne journée à vous! Au revoir! \n\n###STOP###", "final_agent_message": "{\"message\":\"Bonjour Amelia,\\n\\n✅ Les réservations 8C8K4E et LU15PA ont bien été annulées. Les remboursements seront crédités sur les cartes-cadeaux utilisées (gift_card_1647044 et gift_card_7397998) dans un délai de 5 à 7 jours ouvrables.\\n\\n❗ Pour les autres réservations, les règles tarifaires ne …" }, "state_tokens": 276, "tier": "common", "primary": "success", "questions": [ { "id": "success", "type": "noul", "instructions": "Given `customer_goal`, `db_update_required`, `tool_sequence`, `last_actions`, `last_user_message` and `final_agent_message`, did the agent end the session with the airline database in the correct state and with the required information communicated to the customer, according to airline policy?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7602, "no": 0.2398 }, "probability_of_expected": 0.2398, "proper_score": 0.4221 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.33, "no": 0.67 }, "probability_of_expected": 0.67, "proper_score": 0.8911 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1908, "no": 0.8092 }, "probability_of_expected": 0.8092, "proper_score": 0.9636 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6758, "no": 0.3242 }, "probability_of_expected": 0.3242, "proper_score": 0.5433 }, "OpenJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.9929, "no": 0.0071 }, "probability_of_expected": 0.0071, "proper_score": 0.0142 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.8042, "no": 0.1958 }, "probability_of_expected": 0.1958, "proper_score": 0.3532, "truncated": true } } }, { "id": "db_correct", "type": "noul", "instructions": "Did the session end with the airline database in exactly the state the customer's request and airline policy require (no missing and no extra changes)?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6937, "no": 0.3063 }, "probability_of_expected": 0.3063, "proper_score": 0.5188 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.24, "no": 0.76 }, "probability_of_expected": 0.76, "proper_score": 0.9424 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.2872, "no": 0.7128 }, "probability_of_expected": 0.7128, "proper_score": 0.9175 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.5353, "no": 0.4647 }, "probability_of_expected": 0.4647, "proper_score": 0.7134 }, "OpenJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.9823, "no": 0.0177 }, "probability_of_expected": 0.0177, "proper_score": 0.0352 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6507, "no": 0.3493 }, "probability_of_expected": 0.3493, "proper_score": 0.5765, "truncated": true } } }, { "id": "communicated", "type": "noul", "instructions": "Did the agent tell the customer every piece of information the request required?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.595, "no": 0.405 }, "probability_of_expected": 0.595, "proper_score": 0.836 }, "Jev": { "answered": true, "pick": "no", "correct": false, "probabilities": { "yes": 0.32, "no": 0.68 }, "probability_of_expected": 0.32, "proper_score": 0.5376 }, "JevFish": { "answered": true, "pick": "no", "correct": false, "probabilities": { "yes": 0.1739, "no": 0.8261 }, "probability_of_expected": 0.1739, "proper_score": 0.3175 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.5622, "no": 0.4378 }, "probability_of_expected": 0.5622, "proper_score": 0.8083 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9218, "no": 0.0782 }, "probability_of_expected": 0.9218, "proper_score": 0.9939 }, "System One Scorer": { "answered": true, "pick": "no", "correct": false, "probabilities": { "yes": 0.4759, "no": 0.5241 }, "probability_of_expected": 0.4759, "proper_score": 0.7253 } } } ], "latency_ms": { "decider-2b": 8947.1, "Jev": 161.1, "JevFish": 3116.9, "NanoJev": 670.1, "OpenJev": 393.9, "System One Scorer": 3292.8 } }, { "id": "real-time-and-agents:tau2-528dfaf2791c", "state": { "domain": "airline", "customer_goal": "For your upcoming trip from New York to Chicago, you want to change the passenger to yourself, upgrade it to economy class, and have 3 checked bags.", "db_update_required": true, "n_turns": 22, "tool_sequence": [ "get_user_details", "get_reservation_details", "get_reservation_details", "get_reservation_details", "get_reservation_details", "update_reservation_passengers", "update_reservation_flights", "update_reservation_baggages" ], "last_actions": [ { "name": "update_reservation_flights", "args": "{\"reservation_id\": \"FQ8APE\", \"cabin\": \"economy\", \"flights\": [{\"flight_number\": \"HAT056\", \"date\": \"2024-05-25\"}, …" }, { "name": "update_reservation_baggages", "args": "{\"reservation_id\": \"FQ8APE\", \"total_baggages\": 3, \"nonfree_baggages\": 0, \"payment_id\": \"gift_card_8190333\"}" } ], "last_user_message": "Great, thank you for your help! That’s all I needed. Have a good day. ###STOP###", "final_agent_message": "{\"message\":\"Confirmed — I’ve added 3 checked bags to reservation FQ8APE.\\n\\nUpdated reservation summary:\\n- Reservation: FQ8APE\\n- Route: EWR → ORD (via IAH) on 2024-05-25\\n- Cabin: economy\\n- Passenger: Omar Rossi (1970-06-06)\\n- Checked bags: 3 total (0 non-free)\\n- Payment method on file used …" }, "state_tokens": 356, "tier": "long", "primary": "success", "questions": [ { "id": "success", "type": "noul", "instructions": "Given `customer_goal`, `db_update_required`, `tool_sequence`, `last_actions`, `last_user_message` and `final_agent_message`, did the agent end the session with the airline database in the correct state and with the required information communicated to the customer, according to airline policy?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9171, "no": 0.0829 }, "probability_of_expected": 0.9171, "proper_score": 0.9931 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.66, "no": 0.34 }, "probability_of_expected": 0.66, "proper_score": 0.8844 }, "JevFish": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.558, "no": 0.442 }, "probability_of_expected": 0.558, "proper_score": 0.8046 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.7417, "no": 0.2583 }, "probability_of_expected": 0.7417, "proper_score": 0.9333 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9996, "no": 0.0004 }, "probability_of_expected": 0.9996, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.8471, "no": 0.1529 }, "probability_of_expected": 0.8471, "proper_score": 0.9766, "truncated": true } } }, { "id": "db_correct", "type": "noul", "instructions": "Did the session end with the airline database in exactly the state the customer's request and airline policy require (no missing and no extra changes)?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9494, "no": 0.0506 }, "probability_of_expected": 0.9494, "proper_score": 0.9974 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.57, "no": 0.43 }, "probability_of_expected": 0.57, "proper_score": 0.8151 }, "JevFish": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.703, "no": 0.297 }, "probability_of_expected": 0.703, "proper_score": 0.9118 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.6334, "no": 0.3666 }, "probability_of_expected": 0.6334, "proper_score": 0.8656 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9969, "no": 0.0031 }, "probability_of_expected": 0.9969, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.7861, "no": 0.2139 }, "probability_of_expected": 0.7861, "proper_score": 0.9543, "truncated": true } } }, { "id": "communicated", "type": "noul", "instructions": "Did the agent tell the customer every piece of information the request required?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9471, "no": 0.0529 }, "probability_of_expected": 0.9471, "proper_score": 0.9972 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.74, "no": 0.26 }, "probability_of_expected": 0.74, "proper_score": 0.9324 }, "JevFish": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.5651, "no": 0.4349 }, "probability_of_expected": 0.5651, "proper_score": 0.8108 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.6593, "no": 0.3407 }, "probability_of_expected": 0.6593, "proper_score": 0.8839 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9905, "no": 0.0095 }, "probability_of_expected": 0.9905, "proper_score": 0.9999 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.6751, "no": 0.3249 }, "probability_of_expected": 0.6751, "proper_score": 0.8944, "truncated": true } } } ], "latency_ms": { "decider-2b": 7362.8, "Jev": 161.6, "JevFish": 3063.9, "NanoJev": 708.6, "OpenJev": 330.8, "System One Scorer": 3299.7 } }, { "id": "real-time-and-agents:tau2-7ae3a16c1215", "state": { "domain": "airline", "customer_goal": "You want to know the sum of gift card balances and the sum of certificate balances.\n\nAdditionally, you want to change your recent reservation to the cheapest business round trip without changing the …", "db_update_required": true, "n_turns": 79, "tool_sequence": [ "get_new_reservation_id", "code_evaluation", "get_new_reservation_id", "get_new_reservation_id", "get_new_reservation_id", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation" ], "last_actions": [ { "name": "code_evaluation", "args": "{\"expression\": \"# 1. Inspect Data Models and Reservations\\nprint(f\\\"Data Models Keys: …" }, { "name": "code_evaluation", "args": "{\"expression\": \"import datetime\\n\\n# 1. Retrieve Classes\\nsample_res = db.reservations.get('K1NW8N')\\nReservationClass …" } ], "last_user_message": "No, that’s all I needed. Thanks for your help and for confirming the master card charge! Have a great day. \n\n###STOP###", "final_agent_message": "I have successfully booked three new Business Class reservations for you, Aarav, and Evelyn!\n\n**Reservation Details:**\n* **Flights:**\n * **Outbound:** JFK to SFO (May 26)\n * **Inbound:** SFO to JFK via Seattle (May 28)\n* **Passengers & Reservations:**\n * **Mohamed Silva:** …" }, "state_tokens": 325, "tier": "long", "primary": "success", "questions": [ { "id": "success", "type": "noul", "instructions": "Given `customer_goal`, `db_update_required`, `tool_sequence`, `last_actions`, `last_user_message` and `final_agent_message`, did the agent end the session with the airline database in the correct state and with the required information communicated to the customer, according to airline policy?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.3935, "no": 0.6065 }, "probability_of_expected": 0.6065, "proper_score": 0.8452 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.11, "no": 0.89 }, "probability_of_expected": 0.89, "proper_score": 0.9879 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1648, "no": 0.8352 }, "probability_of_expected": 0.8352, "proper_score": 0.9729 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7718, "no": 0.2282 }, "probability_of_expected": 0.2282, "proper_score": 0.4043 }, "OpenJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7833, "no": 0.2167 }, "probability_of_expected": 0.2167, "proper_score": 0.3864 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6881, "no": 0.3119 }, "probability_of_expected": 0.3119, "proper_score": 0.5265, "truncated": true } } }, { "id": "db_correct", "type": "noul", "instructions": "Did the session end with the airline database in exactly the state the customer's request and airline policy require (no missing and no extra changes)?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.2962, "no": 0.7038 }, "probability_of_expected": 0.7038, "proper_score": 0.9123 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.13, "no": 0.87 }, "probability_of_expected": 0.87, "proper_score": 0.9831 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1945, "no": 0.8055 }, "probability_of_expected": 0.8055, "proper_score": 0.9622 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6567, "no": 0.3433 }, "probability_of_expected": 0.3433, "proper_score": 0.5688 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.2233, "no": 0.7767 }, "probability_of_expected": 0.7767, "proper_score": 0.9501 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6392, "no": 0.3608 }, "probability_of_expected": 0.3608, "proper_score": 0.5914, "truncated": true } } }, { "id": "communicated", "type": "noul", "instructions": "Did the agent tell the customer every piece of information the request required?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.4521, "no": 0.5479 }, "probability_of_expected": 0.5479, "proper_score": 0.7956 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.21, "no": 0.79 }, "probability_of_expected": 0.79, "proper_score": 0.9559 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1038, "no": 0.8962 }, "probability_of_expected": 0.8962, "proper_score": 0.9892 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.627, "no": 0.373 }, "probability_of_expected": 0.373, "proper_score": 0.6068 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.384, "no": 0.616 }, "probability_of_expected": 0.616, "proper_score": 0.8526 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.5251, "no": 0.4749 }, "probability_of_expected": 0.4749, "proper_score": 0.7242, "truncated": true } } } ], "latency_ms": { "decider-2b": 7425.3, "Jev": 159.6, "JevFish": 3227.1, "NanoJev": 713.5, "OpenJev": 377.8, "System One Scorer": 3259.5 } }, { "id": "real-time-and-agents:tau2-be5278c57a4d", "state": { "domain": "airline", "customer_goal": "You booked the flight and you want to change the passenger name on the reservation.", "db_update_required": true, "n_turns": 12, "tool_sequence": [ "get_reservation_details", "update_reservation_passengers" ], "last_actions": [ { "name": "get_reservation_details", "args": "{\"reservation_id\": \"3RK2T9\"}" }, { "name": "update_reservation_passengers", "args": "{\"reservation_id\": \"3RK2T9\", \"passengers\": [{\"first_name\": \"Anya\", \"last_name\": \"Garcia\", \"dob\": \"1992-11-12\"}, …" } ], "last_user_message": "No, that’s all I needed. Thank you for your help! \n\n###STOP###", "final_agent_message": "Perfect! I've successfully updated the passenger name. The second passenger's name has been changed from Mei Lee to Mei Garcia on your reservation 3RK2T9. \n\nYour reservation now shows both passengers as:\n- Anya Garcia\n- Mei Garcia\n\nIs there anything else I can help you with today?" }, "state_tokens": 233, "tier": "common", "primary": "success", "questions": [ { "id": "success", "type": "noul", "instructions": "Given `customer_goal`, `db_update_required`, `tool_sequence`, `last_actions`, `last_user_message` and `final_agent_message`, did the agent end the session with the airline database in the correct state and with the required information communicated to the customer, according to airline policy?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.8877, "no": 0.1123 }, "probability_of_expected": 0.8877, "proper_score": 0.9874 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.65, "no": 0.35 }, "probability_of_expected": 0.65, "proper_score": 0.8775 }, "JevFish": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.6029, "no": 0.3971 }, "probability_of_expected": 0.6029, "proper_score": 0.8423 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.7931, "no": 0.2069 }, "probability_of_expected": 0.7931, "proper_score": 0.9572 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 1.0, "no": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.8628, "no": 0.1372 }, "probability_of_expected": 0.8628, "proper_score": 0.9812 } } }, { "id": "db_correct", "type": "noul", "instructions": "Did the session end with the airline database in exactly the state the customer's request and airline policy require (no missing and no extra changes)?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9274, "no": 0.0726 }, "probability_of_expected": 0.9274, "proper_score": 0.9947 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.63, "no": 0.37 }, "probability_of_expected": 0.63, "proper_score": 0.8631 }, "JevFish": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.7127, "no": 0.2873 }, "probability_of_expected": 0.7127, "proper_score": 0.9175 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.6951, "no": 0.3049 }, "probability_of_expected": 0.6951, "proper_score": 0.9071 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 1.0, "no": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.8081, "no": 0.1919 }, "probability_of_expected": 0.8081, "proper_score": 0.9632 } } }, { "id": "communicated", "type": "noul", "instructions": "Did the agent tell the customer every piece of information the request required?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.8301, "no": 0.1699 }, "probability_of_expected": 0.8301, "proper_score": 0.9711 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.81, "no": 0.19 }, "probability_of_expected": 0.81, "proper_score": 0.9639 }, "JevFish": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.5608, "no": 0.4392 }, "probability_of_expected": 0.5608, "proper_score": 0.8071 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.696, "no": 0.304 }, "probability_of_expected": 0.696, "proper_score": 0.9076 }, "OpenJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.9999, "no": 0.0001 }, "probability_of_expected": 0.9999, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.5946, "no": 0.4054 }, "probability_of_expected": 0.5946, "proper_score": 0.8357 } } } ], "latency_ms": { "decider-2b": 8444.6, "Jev": 181.3, "JevFish": 3186.3, "NanoJev": 655.8, "OpenJev": 337.6, "System One Scorer": 3461.6 } }, { "id": "real-time-and-agents:tau2-d50546d34f5b", "state": { "domain": "airline", "customer_goal": "You want to remove passenger Sophia from your upcoming round trip flights from LAS to DEN, departure May 19, return is May 20.", "db_update_required": true, "n_turns": 36, "tool_sequence": [ "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation", "code_evaluation" ], "last_actions": [ { "name": "code_evaluation", "args": "{\"expression\": \"from datetime import datetime\\nreservation = db.reservations['GV1N64']\\nreservation.cabin = …" }, { "name": "code_evaluation", "args": "{\"expression\": \"reservation = db.reservations['GV1N64']\\nreservation.cabin = 'basic_economy'\\nreservation\"}" } ], "last_user_message": "###STOP###", "final_agent_message": "You're welcome! If you need any more assistance in the future, feel free to reach out. Have a great day and safe travels!" }, "state_tokens": 210, "tier": "common", "primary": "success", "questions": [ { "id": "success", "type": "noul", "instructions": "Given `customer_goal`, `db_update_required`, `tool_sequence`, `last_actions`, `last_user_message` and `final_agent_message`, did the agent end the session with the airline database in the correct state and with the required information communicated to the customer, according to airline policy?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6729, "no": 0.3271 }, "probability_of_expected": 0.3271, "proper_score": 0.5472 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.05, "no": 0.95 }, "probability_of_expected": 0.95, "proper_score": 0.9975 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1777, "no": 0.8223 }, "probability_of_expected": 0.8223, "proper_score": 0.9684 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7033, "no": 0.2967 }, "probability_of_expected": 0.2967, "proper_score": 0.5053 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0009, "no": 0.9991 }, "probability_of_expected": 0.9991, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7497, "no": 0.2503 }, "probability_of_expected": 0.2503, "proper_score": 0.4379 } } }, { "id": "db_correct", "type": "noul", "instructions": "Did the session end with the airline database in exactly the state the customer's request and airline policy require (no missing and no extra changes)?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6834, "no": 0.3166 }, "probability_of_expected": 0.3166, "proper_score": 0.533 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.05, "no": 0.95 }, "probability_of_expected": 0.95, "proper_score": 0.9975 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.3842, "no": 0.6158 }, "probability_of_expected": 0.6158, "proper_score": 0.8524 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.5988, "no": 0.4012 }, "probability_of_expected": 0.4012, "proper_score": 0.6415 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0018, "no": 0.9982 }, "probability_of_expected": 0.9982, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.7315, "no": 0.2685 }, "probability_of_expected": 0.2685, "proper_score": 0.4649 } } }, { "id": "communicated", "type": "noul", "instructions": "Did the agent tell the customer every piece of information the request required?", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.4284, "no": 0.5716 }, "probability_of_expected": 0.5716, "proper_score": 0.8165 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.11, "no": 0.89 }, "probability_of_expected": 0.89, "proper_score": 0.9879 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1228, "no": 0.8772 }, "probability_of_expected": 0.8772, "proper_score": 0.9849 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.6123, "no": 0.3877 }, "probability_of_expected": 0.3877, "proper_score": 0.6251 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0023, "no": 0.9977 }, "probability_of_expected": 0.9977, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.569, "no": 0.431 }, "probability_of_expected": 0.431, "proper_score": 0.6763 } } } ], "latency_ms": { "decider-2b": 6116.7, "Jev": 183.5, "JevFish": 3212.6, "NanoJev": 724.8, "OpenJev": 367.0, "System One Scorer": 3416.4 } } ] }