{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "vitaminc-claim-consistency",
 "family": "safety-and-quality",
 "use_case": "self-consistency-checks",
 "vertical": "media",
 "dataset": "tals/vitaminc",
 "licence": "CC-BY-SA-3.0",
 "provenance": "crowd",
 "state_kind": "text",
 "primitive": "choice",
 "items": 200,
 "scores": {
  "decider-2b": {
   "decisionScore": 82.42,
   "accuracy": 0.715,
   "answered": 200,
   "items": 200
  },
  "Jev": {
   "decisionScore": 88.08,
   "accuracy": 0.83,
   "answered": 200,
   "items": 200
  },
  "JevFish": {
   "decisionScore": 78.08,
   "accuracy": 0.63,
   "answered": 200,
   "items": 200
  },
  "NanoJev": {
   "decisionScore": 60.8,
   "accuracy": 0.505,
   "answered": 200,
   "items": 200
  },
  "OpenJev": {
   "decisionScore": 82.77,
   "accuracy": 0.77,
   "answered": 200,
   "items": 200
  },
  "System One Scorer": {
   "decisionScore": 82.32,
   "accuracy": 0.76,
   "answered": 200,
   "items": 200
  }
 },
 "samples": [
  {
   "id": "safety-and-quality:vitaminc-20151",
   "state": {
    "claim": "Lee Byung-hun 's place of birth is Seongnam .",
    "evidence": "Lee was born in Seongnam , South Korea .",
    "page": "Lee Byung-hun"
   },
   "state_tokens": 42,
   "tier": "common",
   "primary": "verdict",
   "questions": [
    {
     "id": "verdict",
     "type": "choice",
     "instructions": "Using only the evidence (one sentence from the Wikipedia article named in page), what is the verdict on the claim?",
     "options": [
      [
       "supports",
       "The evidence establishes that the claim is true."
      ],
      [
       "refutes",
       "The evidence establishes that the claim is false."
      ],
      [
       "not_enough_info",
       "The evidence does not settle whether the claim is true or false."
      ]
     ],
     "graded": true,
     "expected": "supports",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.9622,
        "not_enough_info": 0.0349,
        "refutes": 0.0029
       },
       "probability_of_expected": 0.9622,
       "proper_score": 0.9987
      },
      "Jev": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 1.0,
        "not_enough_info": 0.0,
        "refutes": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.9811,
        "not_enough_info": 0.0114,
        "refutes": 0.0075
       },
       "probability_of_expected": 0.9811,
       "proper_score": 0.9997
      },
      "NanoJev": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.9269,
        "refutes": 0.0383,
        "not_enough_info": 0.0348
       },
       "probability_of_expected": 0.9269,
       "proper_score": 0.996
      },
      "OpenJev": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.9949,
        "refutes": 0.004,
        "not_enough_info": 0.0011
       },
       "probability_of_expected": 0.9949,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.9677,
        "not_enough_info": 0.0184,
        "refutes": 0.0139
       },
       "probability_of_expected": 0.9677,
       "proper_score": 0.9992
      }
     }
    },
    {
     "id": "supported",
     "type": "noul",
     "instructions": "Using only the evidence, is the claim true?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9616,
        "no": 0.0384
       },
       "probability_of_expected": 0.9616,
       "proper_score": 0.9985
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.98,
        "no": 0.02
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9996
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.966,
        "no": 0.034
       },
       "probability_of_expected": 0.966,
       "proper_score": 0.9988
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.8333,
        "no": 0.1667
       },
       "probability_of_expected": 0.8333,
       "proper_score": 0.9722
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.966,
        "no": 0.034
       },
       "probability_of_expected": 0.966,
       "proper_score": 0.9988
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9056,
        "no": 0.0944
       },
       "probability_of_expected": 0.9056,
       "proper_score": 0.9911
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2102.4,
    "Jev": 201.4,
    "JevFish": 2124.0,
    "NanoJev": 743.3,
    "OpenJev": 393.1,
    "System One Scorer": 1370.6
   }
  },
  {
   "id": "safety-and-quality:vitaminc-26876",
   "state": {
    "claim": "There were 266 confirmed cases of coronavirus in Palestine during the pandemic .",
    "evidence": "10 April*Ibrahim Milhem confirmed 3 new cases , one from Halhul and two cases from Artas for a total of 266 .",
    "page": "2020 coronavirus pandemic in the State of Palestine"
   },
   "state_tokens": 71,
   "tier": "common",
   "primary": "verdict",
   "questions": [
    {
     "id": "verdict",
     "type": "choice",
     "instructions": "Using only the evidence (one sentence from the Wikipedia article named in page), what is the verdict on the claim?",
     "options": [
      [
       "supports",
       "The evidence establishes that the claim is true."
      ],
      [
       "refutes",
       "The evidence establishes that the claim is false."
      ],
      [
       "not_enough_info",
       "The evidence does not settle whether the claim is true or false."
      ]
     ],
     "graded": true,
     "expected": "supports",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.7906,
        "not_enough_info": 0.1869,
        "refutes": 0.0225
       },
       "probability_of_expected": 0.7906,
       "proper_score": 0.9604
      },
      "Jev": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.92,
        "not_enough_info": 0.07,
        "refutes": 0.01
       },
       "probability_of_expected": 0.92,
       "proper_score": 0.9943
      },
      "JevFish": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.8273,
        "not_enough_info": 0.1133,
        "refutes": 0.0594
       },
       "probability_of_expected": 0.8273,
       "proper_score": 0.9769
      },
      "NanoJev": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.8502,
        "not_enough_info": 0.0765,
        "refutes": 0.0733
       },
       "probability_of_expected": 0.8502,
       "proper_score": 0.9832
      },
      "OpenJev": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.992,
        "refutes": 0.0079,
        "not_enough_info": 0.0001
       },
       "probability_of_expected": 0.992,
       "proper_score": 0.9999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "supports",
       "correct": true,
       "probabilities": {
        "supports": 0.8445,
        "not_enough_info": 0.1006,
        "refutes": 0.0549
       },
       "probability_of_expected": 0.8445,
       "proper_score": 0.9813
      }
     }
    },
    {
     "id": "supported",
     "type": "noul",
     "instructions": "Using only the evidence, is the claim true?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.8013,
        "no": 0.1987
       },
       "probability_of_expected": 0.8013,
       "proper_score": 0.9605
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.86,
        "no": 0.14
       },
       "probability_of_expected": 0.86,
       "proper_score": 0.9804
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7135,
        "no": 0.2865
       },
       "probability_of_expected": 0.7135,
       "proper_score": 0.9179
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.6469,
        "no": 0.3531
       },
       "probability_of_expected": 0.6469,
       "proper_score": 0.8753
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9574,
        "no": 0.0426
       },
       "probability_of_expected": 0.9574,
       "proper_score": 0.9982
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7958,
        "no": 0.2042
       },
       "probability_of_expected": 0.7958,
       "proper_score": 0.9583
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2098.4,
    "Jev": 190.8,
    "JevFish": 2416.5,
    "NanoJev": 631.2,
    "OpenJev": 408.8,
    "System One Scorer": 1429.7
   }
  },
  {
   "id": "safety-and-quality:vitaminc-28695",
   "state": {
    "claim": "John Wick : Chapter 3 - Parabellum grossed more than $ 255.5 million worldwide .",
    "evidence": "It has grossed $ 255 million worldwide , surpassing the entire gross of the second film in just 10 days , and received positive reviews from critics , with praise for the fight choreography , visual style , and Reeves ' performance.",
    "page": "John Wick-COLON- Chapter 3 – Parabellum"
   },
   "state_tokens": 99,
   "tier": "common",
   "primary": "verdict",
   "questions": [
    {
     "id": "verdict",
     "type": "choice",
     "instructions": "Using only the evidence (one sentence from the Wikipedia article named in page), what is the verdict on the claim?",
     "options": [
      [
       "supports",
       "The evidence establishes that the claim is true."
      ],
      [
       "refutes",
       "The evidence establishes that the claim is false."
      ],
      [
       "not_enough_info",
       "The evidence does not settle whether the claim is true or false."
      ]
     ],
     "graded": true,
     "expected": "refutes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.531,
        "not_enough_info": 0.3793,
        "refutes": 0.0897
       },
       "probability_of_expected": 0.0897,
       "proper_score": 0.3728
      },
      "Jev": {
       "answered": true,
       "pick": "refutes",
       "correct": true,
       "probabilities": {
        "refutes": 0.93,
        "not_enough_info": 0.06,
        "supports": 0.01
       },
       "probability_of_expected": 0.93,
       "proper_score": 0.9957
      },
      "JevFish": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.9319,
        "not_enough_info": 0.0446,
        "refutes": 0.0235
       },
       "probability_of_expected": 0.0235,
       "proper_score": 0.088
      },
      "NanoJev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.9311,
        "refutes": 0.042,
        "not_enough_info": 0.027
       },
       "probability_of_expected": 0.042,
       "proper_score": 0.1073
      },
      "OpenJev": {
       "answered": true,
       "pick": "refutes",
       "correct": true,
       "probabilities": {
        "refutes": 0.5288,
        "not_enough_info": 0.3955,
        "supports": 0.0758
       },
       "probability_of_expected": 0.5288,
       "proper_score": 0.8079
      },
      "System One Scorer": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.4903,
        "refutes": 0.3118,
        "not_enough_info": 0.1979
       },
       "probability_of_expected": 0.3118,
       "proper_score": 0.6234
      }
     }
    },
    {
     "id": "supported",
     "type": "noul",
     "instructions": "Using only the evidence, is the claim true?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.488,
        "no": 0.512
       },
       "probability_of_expected": 0.512,
       "proper_score": 0.7619
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.09,
        "no": 0.91
       },
       "probability_of_expected": 0.91,
       "proper_score": 0.9919
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.8705,
        "no": 0.1295
       },
       "probability_of_expected": 0.1295,
       "proper_score": 0.2422
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.7058,
        "no": 0.2942
       },
       "probability_of_expected": 0.2942,
       "proper_score": 0.5019
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0404,
        "no": 0.9596
       },
       "probability_of_expected": 0.9596,
       "proper_score": 0.9984
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.5924,
        "no": 0.4076
       },
       "probability_of_expected": 0.4076,
       "proper_score": 0.649
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2475.5,
    "Jev": 233.5,
    "JevFish": 2309.0,
    "NanoJev": 660.2,
    "OpenJev": 348.4,
    "System One Scorer": 1429.5
   }
  },
  {
   "id": "safety-and-quality:vitaminc-28810",
   "state": {
    "claim": "Nanabhai Haridas & family reside in Malabar Hill .",
    "evidence": "Notable residents include Adi Godrej , the Birla family , Shashi Ruia & family , Pallonji Mistry , Mahesh Jethmalani , the Jindal family , Nanabhai Haridas & family , the Lal family , the Thakkar family , the Shah family , the Karnavat family , among many more.",
    "page": "Malabar Hill"
   },
   "state_tokens": 101,
   "tier": "common",
   "primary": "verdict",
   "questions": [
    {
     "id": "verdict",
     "type": "choice",
     "instructions": "Using only the evidence (one sentence from the Wikipedia article named in page), what is the verdict on the claim?",
     "options": [
      [
       "supports",
       "The evidence establishes that the claim is true."
      ],
      [
       "refutes",
       "The evidence establishes that the claim is false."
      ],
      [
       "not_enough_info",
       "The evidence does not settle whether the claim is true or false."
      ]
     ],
     "graded": true,
     "expected": "not_enough_info",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "not_enough_info",
       "correct": true,
       "probabilities": {
        "not_enough_info": 0.4996,
        "supports": 0.4762,
        "refutes": 0.0242
       },
       "probability_of_expected": 0.4996,
       "proper_score": 0.7611
      },
      "Jev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.99,
        "not_enough_info": 0.01,
        "refutes": 0.0
       },
       "probability_of_expected": 0.01,
       "proper_score": 0.0199
      },
      "JevFish": {
       "answered": true,
       "pick": "not_enough_info",
       "correct": true,
       "probabilities": {
        "not_enough_info": 0.7661,
        "refutes": 0.1513,
        "supports": 0.0826
       },
       "probability_of_expected": 0.7661,
       "proper_score": 0.9578
      },
      "NanoJev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.8034,
        "refutes": 0.122,
        "not_enough_info": 0.0746
       },
       "probability_of_expected": 0.0746,
       "proper_score": 0.2416
      },
      "OpenJev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 1.0,
        "refutes": 0.0,
        "not_enough_info": 0.0
       },
       "probability_of_expected": 0.0,
       "proper_score": 0.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.8409,
        "not_enough_info": 0.1346,
        "refutes": 0.0245
       },
       "probability_of_expected": 0.1346,
       "proper_score": 0.2717
      }
     }
    },
    {
     "id": "supported",
     "type": "noul",
     "instructions": "Using only the evidence, is the claim true?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": false,
     "expected": null,
     "responses": {
      "decider-2b": {
       "answered": true
      },
      "Jev": {
       "answered": true
      },
      "JevFish": {
       "answered": true
      },
      "NanoJev": {
       "answered": true
      },
      "OpenJev": {
       "answered": true
      },
      "System One Scorer": {
       "answered": true
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2100.5,
    "Jev": 197.8,
    "JevFish": 2330.6,
    "NanoJev": 672.7,
    "OpenJev": 291.2,
    "System One Scorer": 1414.9
   }
  },
  {
   "id": "safety-and-quality:vitaminc-29724",
   "state": {
    "claim": "Following the confirmation of multiple COVID-19 cases in North Rhine-Westphalia , the district of Heinsberg initiated the closure of schools .",
    "evidence": "On 26 February , following the confirmation of multiple COVID-19 cases in North Rhine-Westphalia , Heinsberg initiated closure of schools , swimming pools , libraries and the town hall until 2 March .",
    "page": "2020 coronavirus pandemic in Germany"
   },
   "state_tokens": 94,
   "tier": "common",
   "primary": "verdict",
   "questions": [
    {
     "id": "verdict",
     "type": "choice",
     "instructions": "Using only the evidence (one sentence from the Wikipedia article named in page), what is the verdict on the claim?",
     "options": [
      [
       "supports",
       "The evidence establishes that the claim is true."
      ],
      [
       "refutes",
       "The evidence establishes that the claim is false."
      ],
      [
       "not_enough_info",
       "The evidence does not settle whether the claim is true or false."
      ]
     ],
     "graded": true,
     "expected": "not_enough_info",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.98,
        "not_enough_info": 0.0165,
        "refutes": 0.0035
       },
       "probability_of_expected": 0.0165,
       "proper_score": 0.0362
      },
      "Jev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 1.0,
        "not_enough_info": 0.0,
        "refutes": 0.0
       },
       "probability_of_expected": 0.0,
       "proper_score": 0.0
      },
      "JevFish": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.554,
        "not_enough_info": 0.3801,
        "refutes": 0.0659
       },
       "probability_of_expected": 0.3801,
       "proper_score": 0.6522
      },
      "NanoJev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.9185,
        "not_enough_info": 0.0429,
        "refutes": 0.0386
       },
       "probability_of_expected": 0.0429,
       "proper_score": 0.1194
      },
      "OpenJev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.9999,
        "refutes": 0.0001,
        "not_enough_info": 0.0
       },
       "probability_of_expected": 0.0,
       "proper_score": 0.0001
      },
      "System One Scorer": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.9461,
        "not_enough_info": 0.0317,
        "refutes": 0.0223
       },
       "probability_of_expected": 0.0317,
       "proper_score": 0.0833
      }
     }
    },
    {
     "id": "supported",
     "type": "noul",
     "instructions": "Using only the evidence, is the claim true?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": false,
     "expected": null,
     "responses": {
      "decider-2b": {
       "answered": true
      },
      "Jev": {
       "answered": true
      },
      "JevFish": {
       "answered": true
      },
      "NanoJev": {
       "answered": true
      },
      "OpenJev": {
       "answered": true
      },
      "System One Scorer": {
       "answered": true
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2100.4,
    "Jev": 156.5,
    "JevFish": 2302.0,
    "NanoJev": 681.9,
    "OpenJev": 288.1,
    "System One Scorer": 1248.9
   }
  },
  {
   "id": "safety-and-quality:vitaminc-8120",
   "state": {
    "claim": "Angel Olsen sings and plays folk and indie rock music .",
    "evidence": "Angel Olsen is an American indie pop singer and guitarist , who was raised in St. Louis , Missouri , and is now based in Chicago , Illinois .",
    "page": "Angel Olsen"
   },
   "state_tokens": 54,
   "tier": "common",
   "primary": "verdict",
   "questions": [
    {
     "id": "verdict",
     "type": "choice",
     "instructions": "Using only the evidence (one sentence from the Wikipedia article named in page), what is the verdict on the claim?",
     "options": [
      [
       "supports",
       "The evidence establishes that the claim is true."
      ],
      [
       "refutes",
       "The evidence establishes that the claim is false."
      ],
      [
       "not_enough_info",
       "The evidence does not settle whether the claim is true or false."
      ]
     ],
     "graded": true,
     "expected": "refutes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "not_enough_info",
       "correct": false,
       "probabilities": {
        "not_enough_info": 0.8982,
        "supports": 0.0774,
        "refutes": 0.0244
       },
       "probability_of_expected": 0.0244,
       "proper_score": 0.1177
      },
      "Jev": {
       "answered": true,
       "pick": "not_enough_info",
       "correct": false,
       "probabilities": {
        "not_enough_info": 0.89,
        "refutes": 0.07,
        "supports": 0.04
       },
       "probability_of_expected": 0.07,
       "proper_score": 0.1707
      },
      "JevFish": {
       "answered": true,
       "pick": "not_enough_info",
       "correct": false,
       "probabilities": {
        "not_enough_info": 0.763,
        "refutes": 0.2151,
        "supports": 0.0219
       },
       "probability_of_expected": 0.2151,
       "proper_score": 0.4007
      },
      "NanoJev": {
       "answered": true,
       "pick": "supports",
       "correct": false,
       "probabilities": {
        "supports": 0.948,
        "refutes": 0.0283,
        "not_enough_info": 0.0237
       },
       "probability_of_expected": 0.0283,
       "proper_score": 0.0783
      },
      "OpenJev": {
       "answered": true,
       "pick": "not_enough_info",
       "correct": false,
       "probabilities": {
        "not_enough_info": 0.9609,
        "refutes": 0.0266,
        "supports": 0.0124
       },
       "probability_of_expected": 0.0266,
       "proper_score": 0.0645
      },
      "System One Scorer": {
       "answered": true,
       "pick": "not_enough_info",
       "correct": false,
       "probabilities": {
        "not_enough_info": 0.561,
        "refutes": 0.3362,
        "supports": 0.1028
       },
       "probability_of_expected": 0.3362,
       "proper_score": 0.6171
      }
     }
    },
    {
     "id": "supported",
     "type": "noul",
     "instructions": "Using only the evidence, is the claim true?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.512,
        "no": 0.488
       },
       "probability_of_expected": 0.488,
       "proper_score": 0.7379
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.25,
        "no": 0.75
       },
       "probability_of_expected": 0.75,
       "proper_score": 0.9375
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1078,
        "no": 0.8922
       },
       "probability_of_expected": 0.8922,
       "proper_score": 0.9884
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.7853,
        "no": 0.2147
       },
       "probability_of_expected": 0.2147,
       "proper_score": 0.3833
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0317,
        "no": 0.9683
       },
       "probability_of_expected": 0.9683,
       "proper_score": 0.999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.3589,
        "no": 0.6411
       },
       "probability_of_expected": 0.6411,
       "proper_score": 0.8712
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2096.2,
    "Jev": 141.3,
    "JevFish": 2357.6,
    "NanoJev": 613.9,
    "OpenJev": 383.9,
    "System One Scorer": 1376.3
   }
  }
 ]
}