{ "schemaVersion": "blobfish.typed-decision-bench-task.v1", "version": "0.3.0", "task": "hwu64-tool-select", "family": "workflow-control", "use_case": "typed-tool-dispatch", "vertical": "consumer-assistant", "dataset": "deutsche-telekom/NLU-Evaluation-Data-en-de", "licence": "CC-BY-4.0", "provenance": "crowd", "state_kind": "record", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 91.17, "accuracy": 0.86, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 93.66, "accuracy": 0.88, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 90.22, "accuracy": 0.85, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 67.8, "accuracy": 0.36, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 91.45, "accuracy": 0.87, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 83.37, "accuracy": 0.805, "answered": 200, "items": 200 } }, "samples": [ { "id": "workflow-control:hwu-18d4d0349a5b", "state": { "utterance": "you have gotten my instructions perfectly." }, "state_tokens": 12, "tier": "common", "primary": "tool", "questions": [ { "id": "tool", "type": "choice", "instructions": "Read the spoken command in `utterance`. Choose the single assistant tool that should handle it.", "options": [ [ "alarm_query", "Ask what alarms are set or when an alarm will ring" ], [ "alarm_remove", "Cancel or delete an alarm" ], [ "alarm_set", "Create a new alarm or wake-up call" ], [ "audio_volume_down", "Lower the playback volume" ], [ "audio_volume_mute", "Mute or silence the audio" ], [ "audio_volume_up", "Raise the playback volume" ], [ "calendar_query", "Ask about existing calendar events, appointments or schedule" ], [ "calendar_remove", "Delete or cancel a calendar event" ], [ "calendar_set", "Add a calendar event, meeting or reminder" ], [ "cooking_recipe", "Ask for a recipe or how to cook a dish" ], [ "datetime_convert", "Convert a time between time zones" ], [ "datetime_query", "Ask the current time or date, or what day a date falls on" ], [ "email_addcontact", "Add a new contact to the address book" ], [ "email_query", "Check the inbox, read or search emails" ], [ "email_querycontact", "Look up a contact's details such as their email or phone number" ], [ "email_sendemail", "Compose and send an email or reply" ], [ "general_affirm", "Say yes or agree with the assistant" ], [ "general_commandstop", "Tell the assistant to stop what it is doing" ], [ "general_confirm", "Ask the assistant to confirm it did or understood something" ], [ "general_dontcare", "Express indifference, any option is fine" ], [ "general_explain", "Ask the assistant to explain or clarify what it said" ], [ "general_joke", "Ask the assistant to tell a joke" ], [ "general_negate", "Say no, deny or correct the assistant" ], [ "general_praise", "Praise or thank the assistant" ], [ "general_quirky", "Small talk or a quirky personal question to the assistant" ], [ "general_repeat", "Ask the assistant to repeat what it said" ], [ "iot_cleaning", "Start or control the robot vacuum cleaner" ], [ "iot_coffee", "Make coffee with the connected coffee machine" ], [ "iot_hue_lightchange", "Change the colour of the smart lights" ], [ "iot_hue_lightdim", "Dim the smart lights" ], [ "iot_hue_lightoff", "Turn the smart lights off" ], [ "iot_hue_lighton", "Turn the smart lights on" ], [ "iot_hue_lightup", "Brighten the smart lights" ], [ "iot_wemo_off", "Switch off a smart plug or socket" ], [ "iot_wemo_on", "Switch on a smart plug or socket" ], [ "lists_createoradd", "Create a list or add an item to a list" ], [ "lists_query", "Read out or check what is on a list" ], [ "lists_remove", "Remove an item from a list or delete a list" ], [ "music_likeness", "Say you like or dislike the current song or artist, or save it" ], [ "music_query", "Ask what song or artist is playing or about music details" ], [ "music_settings", "Change music playback settings such as shuffle or repeat" ], [ "news_query", "Ask for news or headlines on a topic" ], [ "play_audiobook", "Play or resume an audiobook" ], [ "play_game", "Start or play a game" ], [ "play_music", "Play a song, artist, album, genre or playlist" ], [ "play_podcasts", "Play or resume a podcast episode" ], [ "play_radio", "Play a radio station" ], [ "qa_currency", "Ask about a currency exchange rate or conversion" ], [ "qa_definition", "Ask the definition or meaning of a word" ], [ "qa_factoid", "Ask a general factual question" ], [ "qa_maths", "Ask for an arithmetic calculation" ], [ "qa_stock", "Ask about a stock price or share" ], [ "recommendation_events", "Ask for events happening nearby or recommendations of things to do" ], [ "recommendation_locations", "Ask for a recommended place such as a restaurant, shop or venue" ], [ "recommendation_movies", "Ask for a movie or show recommendation" ], [ "social_post", "Post, tweet or share something on social media" ], [ "social_query", "Check social media notifications, messages or feeds" ], [ "takeaway_order", "Order takeaway food or a delivery" ], [ "takeaway_query", "Ask about a takeaway order status or whether a place delivers" ], [ "transport_query", "Ask about train, bus or flight times, routes or directions" ], [ "transport_taxi", "Book or call a taxi or ride" ], [ "transport_ticket", "Book or buy a travel ticket" ], [ "transport_traffic", "Ask about traffic conditions on a route" ], [ "weather_query", "Ask about the weather or forecast" ] ], "graded": true, "expected": "general_affirm", "responses": { "decider-2b": { "answered": true, "pick": "general_affirm", "correct": true, "probabilities": { "general_affirm": 0.6646, "general_praise": 0.1729, "general_confirm": 0.136, "general_repeat": 0.0025, "general_commandstop": 0.0022, "calendar_query": 0.0021 }, "probability_of_expected": 0.6646, "proper_score": 0.9195 }, "Jev": { "answered": true, "pick": "general_praise", "correct": false, "probabilities": { "general_praise": 0.87, "general_confirm": 0.12, "general_affirm": 0.01, "iot_hue_lightup": 0.0, "transport_query": 0.0, "social_post": 0.0 }, "probability_of_expected": 0.01, "proper_score": 0.1243 }, "JevFish": { "answered": true, "pick": "general_confirm", "correct": false, "probabilities": { "general_confirm": 0.5186, "general_praise": 0.1803, "general_affirm": 0.1132, "general_repeat": 0.0593, "general_explain": 0.0136, "general_quirky": 0.0086 }, "probability_of_expected": 0.1132, "proper_score": 0.454 }, "NanoJev": { "answered": true, "pick": "general_affirm", "correct": true, "probabilities": { "general_affirm": 0.4635, "general_explain": 0.146, "general_confirm": 0.1084, "qa_factoid": 0.0502, "recommendation_locations": 0.041, "general_repeat": 0.0342 }, "probability_of_expected": 0.4635, "proper_score": 0.8362 }, "OpenJev": { "answered": true, "pick": "general_praise", "correct": false, "probabilities": { "general_praise": 0.9989, "qa_factoid": 0.0007, "play_audiobook": 0.0001, "general_quirky": 0.0001, "general_repeat": 0.0, "alarm_query": 0.0, "general_affirm": 0.0 }, "probability_of_expected": 0.0, "proper_score": 0.0011 }, "System One Scorer": { "answered": true, "pick": "general_confirm", "correct": false, "probabilities": { "general_confirm": 0.2699, "general_praise": 0.1885, "general_affirm": 0.0989, "transport_taxi": 0.0274, "general_commandstop": 0.0256, "lists_query": 0.0209 }, "probability_of_expected": 0.0989, "proper_score": 0.5373 } } }, { "id": "date_mentioned", "type": "noul", "instructions": "For the same command: Answer yes if the utterance explicitly states a day or calendar date for the `date` argument, such as 'tomorrow', 'on friday', 'next week', 'the 3rd of march', 'this weekend', 'in two days'. Answer no if it gives no day or date, including when it only gives a clock time ('at 5am') or a part of the day ('tonight', 'this morning').", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.013, "no": 0.987 }, "probability_of_expected": 0.987, "proper_score": 0.9998 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.05, "no": 0.95 }, "probability_of_expected": 0.95, "proper_score": 0.9975 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1313, "no": 0.8687 }, "probability_of_expected": 0.8687, "proper_score": 0.9828 }, "NanoJev": { "answered": true, "pick": "yes", "correct": false, "probabilities": { "yes": 0.5051, "no": 0.4949 }, "probability_of_expected": 0.4949, "proper_score": 0.7448 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0, "no": 1.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.2528, "no": 0.7472 }, "probability_of_expected": 0.7472, "proper_score": 0.9361 } } } ], "latency_ms": { "decider-2b": 11446.7, "Jev": 279.2, "JevFish": 2745.4, "NanoJev": 2231.3, "OpenJev": 309.7, "System One Scorer": 7506.4 } }, { "id": "workflow-control:hwu-34ce521e9e59", "state": { "utterance": "I'd like to continue to the next episode." }, "state_tokens": 15, "tier": "common", "primary": "tool", "questions": [ { "id": "tool", "type": "choice", "instructions": "Read the spoken command in `utterance`. Choose the single assistant tool that should handle it.", "options": [ [ "alarm_query", "Ask what alarms are set or when an alarm will ring" ], [ "alarm_remove", "Cancel or delete an alarm" ], [ "alarm_set", "Create a new alarm or wake-up call" ], [ "audio_volume_down", "Lower the playback volume" ], [ "audio_volume_mute", "Mute or silence the audio" ], [ "audio_volume_up", "Raise the playback volume" ], [ "calendar_query", "Ask about existing calendar events, appointments or schedule" ], [ "calendar_remove", "Delete or cancel a calendar event" ], [ "calendar_set", "Add a calendar event, meeting or reminder" ], [ "cooking_recipe", "Ask for a recipe or how to cook a dish" ], [ "datetime_convert", "Convert a time between time zones" ], [ "datetime_query", "Ask the current time or date, or what day a date falls on" ], [ "email_addcontact", "Add a new contact to the address book" ], [ "email_query", "Check the inbox, read or search emails" ], [ "email_querycontact", "Look up a contact's details such as their email or phone number" ], [ "email_sendemail", "Compose and send an email or reply" ], [ "general_affirm", "Say yes or agree with the assistant" ], [ "general_commandstop", "Tell the assistant to stop what it is doing" ], [ "general_confirm", "Ask the assistant to confirm it did or understood something" ], [ "general_dontcare", "Express indifference, any option is fine" ], [ "general_explain", "Ask the assistant to explain or clarify what it said" ], [ "general_joke", "Ask the assistant to tell a joke" ], [ "general_negate", "Say no, deny or correct the assistant" ], [ "general_praise", "Praise or thank the assistant" ], [ "general_quirky", "Small talk or a quirky personal question to the assistant" ], [ "general_repeat", "Ask the assistant to repeat what it said" ], [ "iot_cleaning", "Start or control the robot vacuum cleaner" ], [ "iot_coffee", "Make coffee with the connected coffee machine" ], [ "iot_hue_lightchange", "Change the colour of the smart lights" ], [ "iot_hue_lightdim", "Dim the smart lights" ], [ "iot_hue_lightoff", "Turn the smart lights off" ], [ "iot_hue_lighton", "Turn the smart lights on" ], [ "iot_hue_lightup", "Brighten the smart lights" ], [ "iot_wemo_off", "Switch off a smart plug or socket" ], [ "iot_wemo_on", "Switch on a smart plug or socket" ], [ "lists_createoradd", "Create a list or add an item to a list" ], [ "lists_query", "Read out or check what is on a list" ], [ "lists_remove", "Remove an item from a list or delete a list" ], [ "music_likeness", "Say you like or dislike the current song or artist, or save it" ], [ "music_query", "Ask what song or artist is playing or about music details" ], [ "music_settings", "Change music playback settings such as shuffle or repeat" ], [ "news_query", "Ask for news or headlines on a topic" ], [ "play_audiobook", "Play or resume an audiobook" ], [ "play_game", "Start or play a game" ], [ "play_music", "Play a song, artist, album, genre or playlist" ], [ "play_podcasts", "Play or resume a podcast episode" ], [ "play_radio", "Play a radio station" ], [ "qa_currency", "Ask about a currency exchange rate or conversion" ], [ "qa_definition", "Ask the definition or meaning of a word" ], [ "qa_factoid", "Ask a general factual question" ], [ "qa_maths", "Ask for an arithmetic calculation" ], [ "qa_stock", "Ask about a stock price or share" ], [ "recommendation_events", "Ask for events happening nearby or recommendations of things to do" ], [ "recommendation_locations", "Ask for a recommended place such as a restaurant, shop or venue" ], [ "recommendation_movies", "Ask for a movie or show recommendation" ], [ "social_post", "Post, tweet or share something on social media" ], [ "social_query", "Check social media notifications, messages or feeds" ], [ "takeaway_order", "Order takeaway food or a delivery" ], [ "takeaway_query", "Ask about a takeaway order status or whether a place delivers" ], [ "transport_query", "Ask about train, bus or flight times, routes or directions" ], [ "transport_taxi", "Book or call a taxi or ride" ], [ "transport_ticket", "Book or buy a travel ticket" ], [ "transport_traffic", "Ask about traffic conditions on a route" ], [ "weather_query", "Ask about the weather or forecast" ] ], "graded": true, "expected": "play_podcasts", "responses": { "decider-2b": { "answered": true, "pick": "play_podcasts", "correct": true, "probabilities": { "play_podcasts": 0.9733, "play_audiobook": 0.0189, "general_confirm": 0.0022, "general_affirm": 0.0017, "play_music": 0.001, "general_dontcare": 0.0006 }, "probability_of_expected": 0.9733, "proper_score": 0.9995 }, "Jev": { "answered": true, "pick": "play_podcasts", "correct": true, "probabilities": { "play_podcasts": 1.0, "transport_query": 0.0, "audio_volume_mute": 0.0, "general_confirm": 0.0, "transport_traffic": 0.0, "iot_hue_lightchange": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "JevFish": { "answered": true, "pick": "play_podcasts", "correct": true, "probabilities": { "play_podcasts": 0.9784, "play_audiobook": 0.0065, "general_dontcare": 0.0019, "alarm_query": 0.0007, "social_query": 0.0006, "general_affirm": 0.0006 }, "probability_of_expected": 0.9784, "proper_score": 0.9997 }, "NanoJev": { "answered": true, "pick": "general_affirm", "correct": false, "probabilities": { "general_affirm": 0.2244, "general_explain": 0.1238, "qa_factoid": 0.0805, "general_repeat": 0.0609, "recommendation_events": 0.0554, "recommendation_locations": 0.0541, "play_podcasts": 0.0351 }, "probability_of_expected": 0.0351, "proper_score": 0.4894 }, "OpenJev": { "answered": true, "pick": "play_audiobook", "correct": false, "probabilities": { "play_audiobook": 0.999, "play_podcasts": 0.0005, "iot_cleaning": 0.0003, "general_affirm": 0.0, "play_game": 0.0, "music_query": 0.0 }, "probability_of_expected": 0.0005, "proper_score": 0.0015 }, "System One Scorer": { "answered": true, "pick": "play_podcasts", "correct": true, "probabilities": { "play_podcasts": 0.3653, "play_audiobook": 0.1509, "transport_taxi": 0.0306, "general_confirm": 0.0304, "lists_query": 0.0279, "play_game": 0.0174 }, "probability_of_expected": 0.3653, "proper_score": 0.7841 } } }, { "id": "date_mentioned", "type": "noul", "instructions": "For the same command: Answer yes if the utterance explicitly states a day or calendar date for the `date` argument, such as 'tomorrow', 'on friday', 'next week', 'the 3rd of march', 'this weekend', 'in two days'. Answer no if it gives no day or date, including when it only gives a clock time ('at 5am') or a part of the day ('tonight', 'this morning').", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0074, "no": 0.9926 }, "probability_of_expected": 0.9926, "proper_score": 0.9999 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.07, "no": 0.93 }, "probability_of_expected": 0.93, "proper_score": 0.9951 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0981, "no": 0.9019 }, "probability_of_expected": 0.9019, "proper_score": 0.9904 }, "NanoJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.4855, "no": 0.5145 }, "probability_of_expected": 0.5145, "proper_score": 0.7643 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0, "no": 1.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1792, "no": 0.8208 }, "probability_of_expected": 0.8208, "proper_score": 0.9679 } } } ], "latency_ms": { "decider-2b": 11553.9, "Jev": 236.4, "JevFish": 2740.0, "NanoJev": 2232.2, "OpenJev": 331.2, "System One Scorer": 7673.5 } }, { "id": "workflow-control:hwu-a313550076b2", "state": { "utterance": "olly that is all, just stop it." }, "state_tokens": 14, "tier": "common", "primary": "tool", "questions": [ { "id": "tool", "type": "choice", "instructions": "Read the spoken command in `utterance`. Choose the single assistant tool that should handle it.", "options": [ [ "alarm_query", "Ask what alarms are set or when an alarm will ring" ], [ "alarm_remove", "Cancel or delete an alarm" ], [ "alarm_set", "Create a new alarm or wake-up call" ], [ "audio_volume_down", "Lower the playback volume" ], [ "audio_volume_mute", "Mute or silence the audio" ], [ "audio_volume_up", "Raise the playback volume" ], [ "calendar_query", "Ask about existing calendar events, appointments or schedule" ], [ "calendar_remove", "Delete or cancel a calendar event" ], [ "calendar_set", "Add a calendar event, meeting or reminder" ], [ "cooking_recipe", "Ask for a recipe or how to cook a dish" ], [ "datetime_convert", "Convert a time between time zones" ], [ "datetime_query", "Ask the current time or date, or what day a date falls on" ], [ "email_addcontact", "Add a new contact to the address book" ], [ "email_query", "Check the inbox, read or search emails" ], [ "email_querycontact", "Look up a contact's details such as their email or phone number" ], [ "email_sendemail", "Compose and send an email or reply" ], [ "general_affirm", "Say yes or agree with the assistant" ], [ "general_commandstop", "Tell the assistant to stop what it is doing" ], [ "general_confirm", "Ask the assistant to confirm it did or understood something" ], [ "general_dontcare", "Express indifference, any option is fine" ], [ "general_explain", "Ask the assistant to explain or clarify what it said" ], [ "general_joke", "Ask the assistant to tell a joke" ], [ "general_negate", "Say no, deny or correct the assistant" ], [ "general_praise", "Praise or thank the assistant" ], [ "general_quirky", "Small talk or a quirky personal question to the assistant" ], [ "general_repeat", "Ask the assistant to repeat what it said" ], [ "iot_cleaning", "Start or control the robot vacuum cleaner" ], [ "iot_coffee", "Make coffee with the connected coffee machine" ], [ "iot_hue_lightchange", "Change the colour of the smart lights" ], [ "iot_hue_lightdim", "Dim the smart lights" ], [ "iot_hue_lightoff", "Turn the smart lights off" ], [ "iot_hue_lighton", "Turn the smart lights on" ], [ "iot_hue_lightup", "Brighten the smart lights" ], [ "iot_wemo_off", "Switch off a smart plug or socket" ], [ "iot_wemo_on", "Switch on a smart plug or socket" ], [ "lists_createoradd", "Create a list or add an item to a list" ], [ "lists_query", "Read out or check what is on a list" ], [ "lists_remove", "Remove an item from a list or delete a list" ], [ "music_likeness", "Say you like or dislike the current song or artist, or save it" ], [ "music_query", "Ask what song or artist is playing or about music details" ], [ "music_settings", "Change music playback settings such as shuffle or repeat" ], [ "news_query", "Ask for news or headlines on a topic" ], [ "play_audiobook", "Play or resume an audiobook" ], [ "play_game", "Start or play a game" ], [ "play_music", "Play a song, artist, album, genre or playlist" ], [ "play_podcasts", "Play or resume a podcast episode" ], [ "play_radio", "Play a radio station" ], [ "qa_currency", "Ask about a currency exchange rate or conversion" ], [ "qa_definition", "Ask the definition or meaning of a word" ], [ "qa_factoid", "Ask a general factual question" ], [ "qa_maths", "Ask for an arithmetic calculation" ], [ "qa_stock", "Ask about a stock price or share" ], [ "recommendation_events", "Ask for events happening nearby or recommendations of things to do" ], [ "recommendation_locations", "Ask for a recommended place such as a restaurant, shop or venue" ], [ "recommendation_movies", "Ask for a movie or show recommendation" ], [ "social_post", "Post, tweet or share something on social media" ], [ "social_query", "Check social media notifications, messages or feeds" ], [ "takeaway_order", "Order takeaway food or a delivery" ], [ "takeaway_query", "Ask about a takeaway order status or whether a place delivers" ], [ "transport_query", "Ask about train, bus or flight times, routes or directions" ], [ "transport_taxi", "Book or call a taxi or ride" ], [ "transport_ticket", "Book or buy a travel ticket" ], [ "transport_traffic", "Ask about traffic conditions on a route" ], [ "weather_query", "Ask about the weather or forecast" ] ], "graded": true, "expected": "general_commandstop", "responses": { "decider-2b": { "answered": true, "pick": "general_commandstop", "correct": true, "probabilities": { "general_commandstop": 0.9578, "audio_volume_mute": 0.0225, "alarm_remove": 0.0065, "audio_volume_down": 0.0011, "general_confirm": 0.0011, "play_game": 0.0009 }, "probability_of_expected": 0.9578, "proper_score": 0.9988 }, "Jev": { "answered": true, "pick": "general_commandstop", "correct": true, "probabilities": { "general_commandstop": 1.0, "play_podcasts": 0.0, "calendar_set": 0.0, "iot_wemo_on": 0.0, "play_radio": 0.0, "audio_volume_mute": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "JevFish": { "answered": true, "pick": "general_commandstop", "correct": true, "probabilities": { "general_commandstop": 0.8396, "general_dontcare": 0.0324, "general_negate": 0.0182, "calendar_remove": 0.0077, "email_sendemail": 0.0072, "general_praise": 0.0065 }, "probability_of_expected": 0.8396, "proper_score": 0.9862 }, "NanoJev": { "answered": true, "pick": "general_commandstop", "correct": true, "probabilities": { "general_commandstop": 0.6198, "lists_remove": 0.1573, "general_explain": 0.038, "audio_volume_mute": 0.0303, "general_affirm": 0.0303, "qa_factoid": 0.0207 }, "probability_of_expected": 0.6198, "proper_score": 0.9131 }, "OpenJev": { "answered": true, "pick": "general_commandstop", "correct": true, "probabilities": { "general_commandstop": 0.9994, "play_audiobook": 0.0002, "play_game": 0.0001, "alarm_query": 0.0001, "iot_cleaning": 0.0, "general_confirm": 0.0 }, "probability_of_expected": 0.9994, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "general_commandstop", "correct": true, "probabilities": { "general_commandstop": 0.5362, "audio_volume_mute": 0.0534, "general_negate": 0.0355, "iot_hue_lightoff": 0.033, "iot_wemo_off": 0.024, "music_likeness": 0.0189 }, "probability_of_expected": 0.5362, "proper_score": 0.8882 } } }, { "id": "date_mentioned", "type": "noul", "instructions": "For the same command: Answer yes if the utterance explicitly states a day or calendar date for the `date` argument, such as 'tomorrow', 'on friday', 'next week', 'the 3rd of march', 'this weekend', 'in two days'. Answer no if it gives no day or date, including when it only gives a clock time ('at 5am') or a part of the day ('tonight', 'this morning').", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.013, "no": 0.987 }, "probability_of_expected": 0.987, "proper_score": 0.9998 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.06, "no": 0.94 }, "probability_of_expected": 0.94, "proper_score": 0.9964 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1188, "no": 0.8812 }, "probability_of_expected": 0.8812, "proper_score": 0.9859 }, "NanoJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.4706, "no": 0.5294 }, "probability_of_expected": 0.5294, "proper_score": 0.7785 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0, "no": 1.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1694, "no": 0.8306 }, "probability_of_expected": 0.8306, "proper_score": 0.9713 } } } ], "latency_ms": { "decider-2b": 11548.1, "Jev": 241.6, "JevFish": 2813.5, "NanoJev": 2351.3, "OpenJev": 311.5, "System One Scorer": 7570.0 } }, { "id": "workflow-control:hwu-bc4a35642d8c", "state": { "utterance": "What will be the weather day after?" }, "state_tokens": 13, "tier": "common", "primary": "tool", "questions": [ { "id": "tool", "type": "choice", "instructions": "Read the spoken command in `utterance`. Choose the single assistant tool that should handle it.", "options": [ [ "alarm_query", "Ask what alarms are set or when an alarm will ring" ], [ "alarm_remove", "Cancel or delete an alarm" ], [ "alarm_set", "Create a new alarm or wake-up call" ], [ "audio_volume_down", "Lower the playback volume" ], [ "audio_volume_mute", "Mute or silence the audio" ], [ "audio_volume_up", "Raise the playback volume" ], [ "calendar_query", "Ask about existing calendar events, appointments or schedule" ], [ "calendar_remove", "Delete or cancel a calendar event" ], [ "calendar_set", "Add a calendar event, meeting or reminder" ], [ "cooking_recipe", "Ask for a recipe or how to cook a dish" ], [ "datetime_convert", "Convert a time between time zones" ], [ "datetime_query", "Ask the current time or date, or what day a date falls on" ], [ "email_addcontact", "Add a new contact to the address book" ], [ "email_query", "Check the inbox, read or search emails" ], [ "email_querycontact", "Look up a contact's details such as their email or phone number" ], [ "email_sendemail", "Compose and send an email or reply" ], [ "general_affirm", "Say yes or agree with the assistant" ], [ "general_commandstop", "Tell the assistant to stop what it is doing" ], [ "general_confirm", "Ask the assistant to confirm it did or understood something" ], [ "general_dontcare", "Express indifference, any option is fine" ], [ "general_explain", "Ask the assistant to explain or clarify what it said" ], [ "general_joke", "Ask the assistant to tell a joke" ], [ "general_negate", "Say no, deny or correct the assistant" ], [ "general_praise", "Praise or thank the assistant" ], [ "general_quirky", "Small talk or a quirky personal question to the assistant" ], [ "general_repeat", "Ask the assistant to repeat what it said" ], [ "iot_cleaning", "Start or control the robot vacuum cleaner" ], [ "iot_coffee", "Make coffee with the connected coffee machine" ], [ "iot_hue_lightchange", "Change the colour of the smart lights" ], [ "iot_hue_lightdim", "Dim the smart lights" ], [ "iot_hue_lightoff", "Turn the smart lights off" ], [ "iot_hue_lighton", "Turn the smart lights on" ], [ "iot_hue_lightup", "Brighten the smart lights" ], [ "iot_wemo_off", "Switch off a smart plug or socket" ], [ "iot_wemo_on", "Switch on a smart plug or socket" ], [ "lists_createoradd", "Create a list or add an item to a list" ], [ "lists_query", "Read out or check what is on a list" ], [ "lists_remove", "Remove an item from a list or delete a list" ], [ "music_likeness", "Say you like or dislike the current song or artist, or save it" ], [ "music_query", "Ask what song or artist is playing or about music details" ], [ "music_settings", "Change music playback settings such as shuffle or repeat" ], [ "news_query", "Ask for news or headlines on a topic" ], [ "play_audiobook", "Play or resume an audiobook" ], [ "play_game", "Start or play a game" ], [ "play_music", "Play a song, artist, album, genre or playlist" ], [ "play_podcasts", "Play or resume a podcast episode" ], [ "play_radio", "Play a radio station" ], [ "qa_currency", "Ask about a currency exchange rate or conversion" ], [ "qa_definition", "Ask the definition or meaning of a word" ], [ "qa_factoid", "Ask a general factual question" ], [ "qa_maths", "Ask for an arithmetic calculation" ], [ "qa_stock", "Ask about a stock price or share" ], [ "recommendation_events", "Ask for events happening nearby or recommendations of things to do" ], [ "recommendation_locations", "Ask for a recommended place such as a restaurant, shop or venue" ], [ "recommendation_movies", "Ask for a movie or show recommendation" ], [ "social_post", "Post, tweet or share something on social media" ], [ "social_query", "Check social media notifications, messages or feeds" ], [ "takeaway_order", "Order takeaway food or a delivery" ], [ "takeaway_query", "Ask about a takeaway order status or whether a place delivers" ], [ "transport_query", "Ask about train, bus or flight times, routes or directions" ], [ "transport_taxi", "Book or call a taxi or ride" ], [ "transport_ticket", "Book or buy a travel ticket" ], [ "transport_traffic", "Ask about traffic conditions on a route" ], [ "weather_query", "Ask about the weather or forecast" ] ], "graded": true, "expected": "weather_query", "responses": { "decider-2b": { "answered": true, "pick": "weather_query", "correct": true, "probabilities": { "weather_query": 0.9278, "datetime_query": 0.0321, "iot_coffee": 0.0052, "general_dontcare": 0.0045, "alarm_remove": 0.0037, "qa_factoid": 0.0026 }, "probability_of_expected": 0.9278, "proper_score": 0.9968 }, "Jev": { "answered": true, "pick": "weather_query", "correct": true, "probabilities": { "weather_query": 1.0, "email_querycontact": 0.0, "iot_wemo_on": 0.0, "cooking_recipe": 0.0, "calendar_set": 0.0, "lists_createoradd": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "JevFish": { "answered": true, "pick": "weather_query", "correct": true, "probabilities": { "weather_query": 0.948, "datetime_query": 0.0047, "lists_remove": 0.003, "takeaway_query": 0.0025, "takeaway_order": 0.0023, "transport_traffic": 0.0017 }, "probability_of_expected": 0.948, "proper_score": 0.9986 }, "NanoJev": { "answered": true, "pick": "weather_query", "correct": true, "probabilities": { "weather_query": 0.525, "qa_factoid": 0.1218, "general_affirm": 0.0964, "general_explain": 0.0704, "general_confirm": 0.0285, "qa_definition": 0.0271 }, "probability_of_expected": 0.525, "proper_score": 0.8711 }, "OpenJev": { "answered": true, "pick": "weather_query", "correct": true, "probabilities": { "weather_query": 0.9986, "play_audiobook": 0.0006, "transport_taxi": 0.0001, "general_affirm": 0.0001, "social_post": 0.0001, "datetime_query": 0.0001 }, "probability_of_expected": 0.9986, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "weather_query", "correct": true, "probabilities": { "weather_query": 0.5832, "qa_factoid": 0.0285, "transport_taxi": 0.0235, "datetime_query": 0.0224, "general_confirm": 0.017, "lists_query": 0.0157 }, "probability_of_expected": 0.5832, "proper_score": 0.9108 } } }, { "id": "date_mentioned", "type": "noul", "instructions": "For the same command: Answer yes if the utterance explicitly states a day or calendar date for the `date` argument, such as 'tomorrow', 'on friday', 'next week', 'the 3rd of march', 'this weekend', 'in two days'. Answer no if it gives no day or date, including when it only gives a clock time ('at 5am') or a part of the day ('tonight', 'this morning').", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "yes", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": false, "probabilities": { "yes": 0.1632, "no": 0.8368 }, "probability_of_expected": 0.1632, "proper_score": 0.2998 }, "Jev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.75, "no": 0.25 }, "probability_of_expected": 0.75, "proper_score": 0.9375 }, "JevFish": { "answered": true, "pick": "no", "correct": false, "probabilities": { "yes": 0.3935, "no": 0.6065 }, "probability_of_expected": 0.3935, "proper_score": 0.6321 }, "NanoJev": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.5266, "no": 0.4734 }, "probability_of_expected": 0.5266, "proper_score": 0.7759 }, "OpenJev": { "answered": true, "pick": "no", "correct": false, "probabilities": { "yes": 0.2726, "no": 0.7274 }, "probability_of_expected": 0.2726, "proper_score": 0.471 }, "System One Scorer": { "answered": true, "pick": "yes", "correct": true, "probabilities": { "yes": 0.584, "no": 0.416 }, "probability_of_expected": 0.584, "proper_score": 0.8269 } } } ], "latency_ms": { "decider-2b": 11391.2, "Jev": 186.3, "JevFish": 2622.8, "NanoJev": 2208.4, "OpenJev": 513.8, "System One Scorer": 7402.7 } }, { "id": "workflow-control:hwu-e3dc5c4d7d18", "state": { "utterance": "Read out my to do list." }, "state_tokens": 12, "tier": "common", "primary": "tool", "questions": [ { "id": "tool", "type": "choice", "instructions": "Read the spoken command in `utterance`. Choose the single assistant tool that should handle it.", "options": [ [ "alarm_query", "Ask what alarms are set or when an alarm will ring" ], [ "alarm_remove", "Cancel or delete an alarm" ], [ "alarm_set", "Create a new alarm or wake-up call" ], [ "audio_volume_down", "Lower the playback volume" ], [ "audio_volume_mute", "Mute or silence the audio" ], [ "audio_volume_up", "Raise the playback volume" ], [ "calendar_query", "Ask about existing calendar events, appointments or schedule" ], [ "calendar_remove", "Delete or cancel a calendar event" ], [ "calendar_set", "Add a calendar event, meeting or reminder" ], [ "cooking_recipe", "Ask for a recipe or how to cook a dish" ], [ "datetime_convert", "Convert a time between time zones" ], [ "datetime_query", "Ask the current time or date, or what day a date falls on" ], [ "email_addcontact", "Add a new contact to the address book" ], [ "email_query", "Check the inbox, read or search emails" ], [ "email_querycontact", "Look up a contact's details such as their email or phone number" ], [ "email_sendemail", "Compose and send an email or reply" ], [ "general_affirm", "Say yes or agree with the assistant" ], [ "general_commandstop", "Tell the assistant to stop what it is doing" ], [ "general_confirm", "Ask the assistant to confirm it did or understood something" ], [ "general_dontcare", "Express indifference, any option is fine" ], [ "general_explain", "Ask the assistant to explain or clarify what it said" ], [ "general_joke", "Ask the assistant to tell a joke" ], [ "general_negate", "Say no, deny or correct the assistant" ], [ "general_praise", "Praise or thank the assistant" ], [ "general_quirky", "Small talk or a quirky personal question to the assistant" ], [ "general_repeat", "Ask the assistant to repeat what it said" ], [ "iot_cleaning", "Start or control the robot vacuum cleaner" ], [ "iot_coffee", "Make coffee with the connected coffee machine" ], [ "iot_hue_lightchange", "Change the colour of the smart lights" ], [ "iot_hue_lightdim", "Dim the smart lights" ], [ "iot_hue_lightoff", "Turn the smart lights off" ], [ "iot_hue_lighton", "Turn the smart lights on" ], [ "iot_hue_lightup", "Brighten the smart lights" ], [ "iot_wemo_off", "Switch off a smart plug or socket" ], [ "iot_wemo_on", "Switch on a smart plug or socket" ], [ "lists_createoradd", "Create a list or add an item to a list" ], [ "lists_query", "Read out or check what is on a list" ], [ "lists_remove", "Remove an item from a list or delete a list" ], [ "music_likeness", "Say you like or dislike the current song or artist, or save it" ], [ "music_query", "Ask what song or artist is playing or about music details" ], [ "music_settings", "Change music playback settings such as shuffle or repeat" ], [ "news_query", "Ask for news or headlines on a topic" ], [ "play_audiobook", "Play or resume an audiobook" ], [ "play_game", "Start or play a game" ], [ "play_music", "Play a song, artist, album, genre or playlist" ], [ "play_podcasts", "Play or resume a podcast episode" ], [ "play_radio", "Play a radio station" ], [ "qa_currency", "Ask about a currency exchange rate or conversion" ], [ "qa_definition", "Ask the definition or meaning of a word" ], [ "qa_factoid", "Ask a general factual question" ], [ "qa_maths", "Ask for an arithmetic calculation" ], [ "qa_stock", "Ask about a stock price or share" ], [ "recommendation_events", "Ask for events happening nearby or recommendations of things to do" ], [ "recommendation_locations", "Ask for a recommended place such as a restaurant, shop or venue" ], [ "recommendation_movies", "Ask for a movie or show recommendation" ], [ "social_post", "Post, tweet or share something on social media" ], [ "social_query", "Check social media notifications, messages or feeds" ], [ "takeaway_order", "Order takeaway food or a delivery" ], [ "takeaway_query", "Ask about a takeaway order status or whether a place delivers" ], [ "transport_query", "Ask about train, bus or flight times, routes or directions" ], [ "transport_taxi", "Book or call a taxi or ride" ], [ "transport_ticket", "Book or buy a travel ticket" ], [ "transport_traffic", "Ask about traffic conditions on a route" ], [ "weather_query", "Ask about the weather or forecast" ] ], "graded": true, "expected": "lists_query", "responses": { "decider-2b": { "answered": true, "pick": "lists_query", "correct": true, "probabilities": { "lists_query": 0.9818, "lists_createoradd": 0.0039, "calendar_query": 0.0024, "iot_hue_lightchange": 0.0018, "play_audiobook": 0.0014, "qa_factoid": 0.0012 }, "probability_of_expected": 0.9818, "proper_score": 0.9998 }, "Jev": { "answered": true, "pick": "lists_query", "correct": true, "probabilities": { "lists_query": 1.0, "qa_currency": 0.0, "play_radio": 0.0, "music_query": 0.0, "calendar_remove": 0.0, "iot_wemo_on": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "JevFish": { "answered": true, "pick": "lists_query", "correct": true, "probabilities": { "lists_query": 0.9804, "play_audiobook": 0.0067, "lists_createoradd": 0.0014, "qa_factoid": 0.0013, "qa_stock": 0.0011, "iot_hue_lightchange": 0.0009 }, "probability_of_expected": 0.9804, "proper_score": 0.9998 }, "NanoJev": { "answered": true, "pick": "lists_query", "correct": true, "probabilities": { "lists_query": 0.5435, "general_affirm": 0.1232, "general_explain": 0.0574, "recommendation_locations": 0.0475, "qa_factoid": 0.0385, "recommendation_events": 0.0356 }, "probability_of_expected": 0.5435, "proper_score": 0.8831 }, "OpenJev": { "answered": true, "pick": "lists_query", "correct": true, "probabilities": { "lists_query": 0.9996, "play_audiobook": 0.0003, "datetime_convert": 0.0001, "lists_createoradd": 0.0, "music_query": 0.0, "lists_remove": 0.0 }, "probability_of_expected": 0.9996, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "lists_query", "correct": true, "probabilities": { "lists_query": 0.5816, "lists_createoradd": 0.0288, "transport_taxi": 0.0288, "general_confirm": 0.02, "recommendation_events": 0.0168, "takeaway_query": 0.0156 }, "probability_of_expected": 0.5816, "proper_score": 0.9101 } } }, { "id": "date_mentioned", "type": "noul", "instructions": "For the same command: Answer yes if the utterance explicitly states a day or calendar date for the `date` argument, such as 'tomorrow', 'on friday', 'next week', 'the 3rd of march', 'this weekend', 'in two days'. Answer no if it gives no day or date, including when it only gives a clock time ('at 5am') or a part of the day ('tonight', 'this morning').", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0093, "no": 0.9907 }, "probability_of_expected": 0.9907, "proper_score": 0.9999 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.06, "no": 0.94 }, "probability_of_expected": 0.94, "proper_score": 0.9964 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1004, "no": 0.8996 }, "probability_of_expected": 0.8996, "proper_score": 0.9899 }, "NanoJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.4888, "no": 0.5112 }, "probability_of_expected": 0.5112, "proper_score": 0.761 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0, "no": 1.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.2338, "no": 0.7662 }, "probability_of_expected": 0.7662, "proper_score": 0.9453 } } } ], "latency_ms": { "decider-2b": 11510.7, "Jev": 310.6, "JevFish": 2761.4, "NanoJev": 2225.3, "OpenJev": 346.1, "System One Scorer": 7223.9 } }, { "id": "workflow-control:hwu-f9002068b8ff", "state": { "utterance": "Please paly for me Elvis Preley song again" }, "state_tokens": 15, "tier": "common", "primary": "tool", "questions": [ { "id": "tool", "type": "choice", "instructions": "Read the spoken command in `utterance`. Choose the single assistant tool that should handle it.", "options": [ [ "alarm_query", "Ask what alarms are set or when an alarm will ring" ], [ "alarm_remove", "Cancel or delete an alarm" ], [ "alarm_set", "Create a new alarm or wake-up call" ], [ "audio_volume_down", "Lower the playback volume" ], [ "audio_volume_mute", "Mute or silence the audio" ], [ "audio_volume_up", "Raise the playback volume" ], [ "calendar_query", "Ask about existing calendar events, appointments or schedule" ], [ "calendar_remove", "Delete or cancel a calendar event" ], [ "calendar_set", "Add a calendar event, meeting or reminder" ], [ "cooking_recipe", "Ask for a recipe or how to cook a dish" ], [ "datetime_convert", "Convert a time between time zones" ], [ "datetime_query", "Ask the current time or date, or what day a date falls on" ], [ "email_addcontact", "Add a new contact to the address book" ], [ "email_query", "Check the inbox, read or search emails" ], [ "email_querycontact", "Look up a contact's details such as their email or phone number" ], [ "email_sendemail", "Compose and send an email or reply" ], [ "general_affirm", "Say yes or agree with the assistant" ], [ "general_commandstop", "Tell the assistant to stop what it is doing" ], [ "general_confirm", "Ask the assistant to confirm it did or understood something" ], [ "general_dontcare", "Express indifference, any option is fine" ], [ "general_explain", "Ask the assistant to explain or clarify what it said" ], [ "general_joke", "Ask the assistant to tell a joke" ], [ "general_negate", "Say no, deny or correct the assistant" ], [ "general_praise", "Praise or thank the assistant" ], [ "general_quirky", "Small talk or a quirky personal question to the assistant" ], [ "general_repeat", "Ask the assistant to repeat what it said" ], [ "iot_cleaning", "Start or control the robot vacuum cleaner" ], [ "iot_coffee", "Make coffee with the connected coffee machine" ], [ "iot_hue_lightchange", "Change the colour of the smart lights" ], [ "iot_hue_lightdim", "Dim the smart lights" ], [ "iot_hue_lightoff", "Turn the smart lights off" ], [ "iot_hue_lighton", "Turn the smart lights on" ], [ "iot_hue_lightup", "Brighten the smart lights" ], [ "iot_wemo_off", "Switch off a smart plug or socket" ], [ "iot_wemo_on", "Switch on a smart plug or socket" ], [ "lists_createoradd", "Create a list or add an item to a list" ], [ "lists_query", "Read out or check what is on a list" ], [ "lists_remove", "Remove an item from a list or delete a list" ], [ "music_likeness", "Say you like or dislike the current song or artist, or save it" ], [ "music_query", "Ask what song or artist is playing or about music details" ], [ "music_settings", "Change music playback settings such as shuffle or repeat" ], [ "news_query", "Ask for news or headlines on a topic" ], [ "play_audiobook", "Play or resume an audiobook" ], [ "play_game", "Start or play a game" ], [ "play_music", "Play a song, artist, album, genre or playlist" ], [ "play_podcasts", "Play or resume a podcast episode" ], [ "play_radio", "Play a radio station" ], [ "qa_currency", "Ask about a currency exchange rate or conversion" ], [ "qa_definition", "Ask the definition or meaning of a word" ], [ "qa_factoid", "Ask a general factual question" ], [ "qa_maths", "Ask for an arithmetic calculation" ], [ "qa_stock", "Ask about a stock price or share" ], [ "recommendation_events", "Ask for events happening nearby or recommendations of things to do" ], [ "recommendation_locations", "Ask for a recommended place such as a restaurant, shop or venue" ], [ "recommendation_movies", "Ask for a movie or show recommendation" ], [ "social_post", "Post, tweet or share something on social media" ], [ "social_query", "Check social media notifications, messages or feeds" ], [ "takeaway_order", "Order takeaway food or a delivery" ], [ "takeaway_query", "Ask about a takeaway order status or whether a place delivers" ], [ "transport_query", "Ask about train, bus or flight times, routes or directions" ], [ "transport_taxi", "Book or call a taxi or ride" ], [ "transport_ticket", "Book or buy a travel ticket" ], [ "transport_traffic", "Ask about traffic conditions on a route" ], [ "weather_query", "Ask about the weather or forecast" ] ], "graded": true, "expected": "play_music", "responses": { "decider-2b": { "answered": true, "pick": "play_music", "correct": true, "probabilities": { "play_music": 0.9717, "general_confirm": 0.0065, "qa_stock": 0.0045, "music_query": 0.003, "general_repeat": 0.0022, "play_game": 0.0022 }, "probability_of_expected": 0.9717, "proper_score": 0.9996 }, "Jev": { "answered": true, "pick": "play_music", "correct": true, "probabilities": { "play_music": 1.0, "general_praise": 0.0, "iot_coffee": 0.0, "recommendation_locations": 0.0, "iot_wemo_off": 0.0, "qa_definition": 0.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "JevFish": { "answered": true, "pick": "music_settings", "correct": false, "probabilities": { "music_settings": 0.8796, "general_repeat": 0.0521, "qa_stock": 0.0282, "play_music": 0.0275, "email_querycontact": 0.002, "play_audiobook": 0.0014 }, "probability_of_expected": 0.0275, "proper_score": 0.1385 }, "NanoJev": { "answered": true, "pick": "play_music", "correct": true, "probabilities": { "play_music": 0.3505, "music_query": 0.2729, "general_affirm": 0.0929, "general_repeat": 0.0639, "general_explain": 0.0419, "music_likeness": 0.0296 }, "probability_of_expected": 0.3505, "proper_score": 0.7432 }, "OpenJev": { "answered": true, "pick": "play_music", "correct": true, "probabilities": { "play_music": 0.9978, "general_confirm": 0.0007, "play_audiobook": 0.0006, "music_likeness": 0.0002, "news_query": 0.0001, "music_query": 0.0001 }, "probability_of_expected": 0.9978, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "play_music", "correct": true, "probabilities": { "play_music": 0.2246, "general_confirm": 0.0491, "music_query": 0.0397, "transport_taxi": 0.0369, "music_likeness": 0.0352, "takeaway_query": 0.0283 }, "probability_of_expected": 0.2246, "proper_score": 0.6918 } } }, { "id": "date_mentioned", "type": "noul", "instructions": "For the same command: Answer yes if the utterance explicitly states a day or calendar date for the `date` argument, such as 'tomorrow', 'on friday', 'next week', 'the 3rd of march', 'this weekend', 'in two days'. Answer no if it gives no day or date, including when it only gives a clock time ('at 5am') or a part of the day ('tonight', 'this morning').", "options": [ [ "yes", "" ], [ "no", "" ] ], "graded": true, "expected": "no", "responses": { "decider-2b": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0064, "no": 0.9936 }, "probability_of_expected": 0.9936, "proper_score": 1.0 }, "Jev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.07, "no": 0.93 }, "probability_of_expected": 0.93, "proper_score": 0.9951 }, "JevFish": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0843, "no": 0.9157 }, "probability_of_expected": 0.9157, "proper_score": 0.9929 }, "NanoJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.4766, "no": 0.5234 }, "probability_of_expected": 0.5234, "proper_score": 0.7729 }, "OpenJev": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.0, "no": 1.0 }, "probability_of_expected": 1.0, "proper_score": 1.0 }, "System One Scorer": { "answered": true, "pick": "no", "correct": true, "probabilities": { "yes": 0.1351, "no": 0.8649 }, "probability_of_expected": 0.8649, "proper_score": 0.9818 } } } ], "latency_ms": { "decider-2b": 11775.2, "Jev": 184.6, "JevFish": 2734.0, "NanoJev": 2258.6, "OpenJev": 426.4, "System One Scorer": 7566.6 } } ] }