{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "humanevalpack-spec-match",
 "family": "safety-and-quality",
 "use_case": "semantic-linting",
 "vertical": "software-engineering",
 "dataset": "bigcode/humanevalpack",
 "licence": "MIT",
 "provenance": "verifier",
 "state_kind": "text",
 "primitive": "noul",
 "items": 164,
 "scores": {
  "decider-2b": {
   "decisionScore": 71.39,
   "accuracy": 0.6524,
   "answered": 164,
   "items": 164
  },
  "Jev": {
   "decisionScore": 82.84,
   "accuracy": 0.8902,
   "answered": 164,
   "items": 164
  },
  "JevFish": {
   "decisionScore": 69.39,
   "accuracy": 0.628,
   "answered": 164,
   "items": 164
  },
  "NanoJev": {
   "decisionScore": 68.07,
   "accuracy": 0.5031,
   "answered": 163,
   "items": 164
  },
  "OpenJev": {
   "decisionScore": 80.53,
   "accuracy": 0.8841,
   "answered": 164,
   "items": 164
  },
  "System One Scorer": {
   "decisionScore": 73.16,
   "accuracy": 0.6585,
   "answered": 164,
   "items": 164
  }
 },
 "samples": [
  {
   "id": "safety-and-quality:heval-Python-112-ok",
   "state": {
    "file": "solution.py",
    "function": "def reverse_delete(s,c):\n    \"\"\"Task\n    We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c\n    then check if the result string is palindrome.\n    A string is called palindrome if it reads the same backward as forward.\n    You should return a tuple containing the result string and True/False for the check.\n    Example\n    For s = \"abcde\", c = \"ae\", the result should be ('bcd',False)\n    For s = \"abcdef\", c = \"b\"  the result should be ('acdef',False)\n    For s = \"abcdedcba\", c = \"ab\", the result should be ('cdedc',True)\n    \"\"\"\n    s = ''.join([char for char in s if char not in c])\n    return (s,s[::-1] == s)"
   },
   "state_tokens": 207,
   "tier": "common",
   "primary": "implements_spec",
   "questions": [
    {
     "id": "implements_spec",
     "type": "noul",
     "instructions": "The state holds a Python function whose docstring states the required behaviour, followed by its body. Does the body correctly implement the docstring for all valid inputs (it would pass a hidden test suite written from the docstring)?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7773,
        "no": 0.2227
       },
       "probability_of_expected": 0.7773,
       "proper_score": 0.9504
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.91,
        "no": 0.09
       },
       "probability_of_expected": 0.91,
       "proper_score": 0.9919
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7127,
        "no": 0.2873
       },
       "probability_of_expected": 0.7127,
       "proper_score": 0.9175
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.561,
        "no": 0.439
       },
       "probability_of_expected": 0.561,
       "proper_score": 0.8073
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9996,
        "no": 0.0004
       },
       "probability_of_expected": 0.9996,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7853,
        "no": 0.2147
       },
       "probability_of_expected": 0.7853,
       "proper_score": 0.9539
      }
     }
    },
    {
     "id": "defect_type",
     "type": "choice",
     "instructions": "Which single kind of defect does the body have relative to its docstring? Pick no_bug if it is correct.",
     "options": [
      [
       "no_bug",
       "The body implements the docstring correctly."
      ],
      [
       "missing_logic",
       "A needed step, condition or case is absent."
      ],
      [
       "excess_logic",
       "An extra step or condition changes the result."
      ],
      [
       "value_misuse",
       "A wrong literal, constant or initial value is used."
      ],
      [
       "operator_misuse",
       "A wrong operator or comparison is used."
      ],
      [
       "variable_misuse",
       "The wrong variable is read or written."
      ],
      [
       "function_misuse",
       "The wrong function or method is called."
      ]
     ],
     "graded": true,
     "expected": "no_bug",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.8238,
        "excess_logic": 0.0507,
        "missing_logic": 0.046,
        "operator_misuse": 0.0271,
        "value_misuse": 0.0246,
        "variable_misuse": 0.0152
       },
       "probability_of_expected": 0.8238,
       "proper_score": 0.9813
      },
      "Jev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 1.0,
        "value_misuse": 0.0,
        "function_misuse": 0.0,
        "excess_logic": 0.0,
        "variable_misuse": 0.0,
        "missing_logic": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.8517,
        "missing_logic": 0.0574,
        "excess_logic": 0.0256,
        "operator_misuse": 0.0211,
        "variable_misuse": 0.0185,
        "function_misuse": 0.0155
       },
       "probability_of_expected": 0.8517,
       "proper_score": 0.9865
      },
      "NanoJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.3081,
        "excess_logic": 0.1702,
        "missing_logic": 0.1257,
        "value_misuse": 0.1118,
        "variable_misuse": 0.0983,
        "operator_misuse": 0.096
       },
       "probability_of_expected": 0.3081,
       "proper_score": 0.7186
      },
      "OpenJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.9999,
        "missing_logic": 0.0001,
        "value_misuse": 0.0,
        "function_misuse": 0.0,
        "excess_logic": 0.0,
        "operator_misuse": 0.0
       },
       "probability_of_expected": 0.9999,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.8319,
        "missing_logic": 0.0573,
        "operator_misuse": 0.037,
        "variable_misuse": 0.0295,
        "value_misuse": 0.0212,
        "excess_logic": 0.0183
       },
       "probability_of_expected": 0.8319,
       "proper_score": 0.9827
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4624.4,
    "Jev": 225.0,
    "JevFish": 2200.6,
    "NanoJev": 873.4,
    "OpenJev": 296.6,
    "System One Scorer": 4217.8
   }
  },
  {
   "id": "safety-and-quality:heval-Python-159-buggy",
   "state": {
    "file": "solution.py",
    "function": "def eat(number, need, remaining):\n    \"\"\"\n    You're a hungry rabbit, and you already have eaten a certain number of carrots,\n    but now you need to eat more carrots to complete the day's meals.\n    you should return an array of [ total number of eaten carrots after your meals,\n                                    the number of carrots left after your meals ]\n    if there are not enough remaining carrots, you will eat all remaining carrots, but will still be hungry.\n    \n    Example:\n    * eat(5, 6, 10) -> [11, 4]\n    * eat(4, 8, 9) -> [12, 1]\n    * eat(1, 10, 10) -> [11, 0]\n    * eat(2, 11, 5) -> [7, 0]\n    \n    Variables:\n    @number : integer\n        the number of carrots that you have eaten.\n    @need : integer\n        the number of carrots that you need to eat.\n    @remaining : integer\n        the number of remaining carrots thet exist in stock\n    \n    Constrain:\n    * 0 <= number <= 1000\n    * 0 <= need <= 1000\n    * 0 <= remaining <= 1000\n\n    Have fun :)\n    \"\"\"\n    if(need <= remaining):\n        return [ number + need , number + remaining-need ]\n    else:\n        return [ …"
   },
   "state_tokens": 356,
   "tier": "long",
   "primary": "implements_spec",
   "questions": [
    {
     "id": "implements_spec",
     "type": "noul",
     "instructions": "The state holds a Python function whose docstring states the required behaviour, followed by its body. Does the body correctly implement the docstring for all valid inputs (it would pass a hidden test suite written from the docstring)?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.464,
        "no": 0.536
       },
       "probability_of_expected": 0.536,
       "proper_score": 0.7847
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.06,
        "no": 0.94
       },
       "probability_of_expected": 0.94,
       "proper_score": 0.9964
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.4472,
        "no": 0.5528
       },
       "probability_of_expected": 0.5528,
       "proper_score": 0.8
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.5846,
        "no": 0.4154
       },
       "probability_of_expected": 0.4154,
       "proper_score": 0.6582
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0098,
        "no": 0.9902
       },
       "probability_of_expected": 0.9902,
       "proper_score": 0.9999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.6802,
        "no": 0.3198
       },
       "probability_of_expected": 0.3198,
       "proper_score": 0.5373,
       "truncated": true
      }
     }
    },
    {
     "id": "defect_type",
     "type": "choice",
     "instructions": "Which single kind of defect does the body have relative to its docstring? Pick no_bug if it is correct.",
     "options": [
      [
       "no_bug",
       "The body implements the docstring correctly."
      ],
      [
       "missing_logic",
       "A needed step, condition or case is absent."
      ],
      [
       "excess_logic",
       "An extra step or condition changes the result."
      ],
      [
       "value_misuse",
       "A wrong literal, constant or initial value is used."
      ],
      [
       "operator_misuse",
       "A wrong operator or comparison is used."
      ],
      [
       "variable_misuse",
       "The wrong variable is read or written."
      ],
      [
       "function_misuse",
       "The wrong function or method is called."
      ]
     ],
     "graded": true,
     "expected": "excess_logic",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "missing_logic",
       "correct": false,
       "probabilities": {
        "missing_logic": 0.2605,
        "no_bug": 0.2482,
        "excess_logic": 0.1952,
        "operator_misuse": 0.186,
        "variable_misuse": 0.0533,
        "value_misuse": 0.0399
       },
       "probability_of_expected": 0.1952,
       "proper_score": 0.5918
      },
      "Jev": {
       "answered": true,
       "pick": "missing_logic",
       "correct": false,
       "probabilities": {
        "missing_logic": 0.79,
        "variable_misuse": 0.14,
        "operator_misuse": 0.04,
        "no_bug": 0.01,
        "value_misuse": 0.01,
        "function_misuse": 0.0,
        "excess_logic": 0.0
       },
       "probability_of_expected": 0.0,
       "proper_score": 0.1772
      },
      "JevFish": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.3209,
        "missing_logic": 0.2916,
        "excess_logic": 0.2844,
        "variable_misuse": 0.0406,
        "operator_misuse": 0.0281,
        "function_misuse": 0.0228
       },
       "probability_of_expected": 0.2844,
       "proper_score": 0.6484
      },
      "NanoJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.2684,
        "excess_logic": 0.1971,
        "value_misuse": 0.1125,
        "operator_misuse": 0.1095,
        "missing_logic": 0.1057,
        "function_misuse": 0.1036
       },
       "probability_of_expected": 0.1971,
       "proper_score": 0.613
      },
      "OpenJev": {
       "answered": true,
       "pick": "missing_logic",
       "correct": false,
       "probabilities": {
        "missing_logic": 0.9614,
        "no_bug": 0.0145,
        "function_misuse": 0.0091,
        "variable_misuse": 0.0059,
        "operator_misuse": 0.0046,
        "value_misuse": 0.003,
        "excess_logic": 0.0015
       },
       "probability_of_expected": 0.0015,
       "proper_score": 0.0391
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.4737,
        "missing_logic": 0.3906,
        "operator_misuse": 0.0413,
        "variable_misuse": 0.0314,
        "value_misuse": 0.031,
        "excess_logic": 0.0283
       },
       "probability_of_expected": 0.0283,
       "proper_score": 0.3375,
       "truncated": true
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4422.2,
    "Jev": 166.1,
    "JevFish": 2245.0,
    "NanoJev": 774.5,
    "OpenJev": 409.0,
    "System One Scorer": 4058.2
   }
  },
  {
   "id": "safety-and-quality:heval-Python-27-ok",
   "state": {
    "file": "solution.py",
    "function": "def flip_case(string: str) -> str:\n    \"\"\" For a given string, flip lowercase characters to uppercase and uppercase to lowercase.\n    >>> flip_case('Hello')\n    'hELLO'\n    \"\"\"\n    return string.swapcase()"
   },
   "state_tokens": 67,
   "tier": "common",
   "primary": "implements_spec",
   "questions": [
    {
     "id": "implements_spec",
     "type": "noul",
     "instructions": "The state holds a Python function whose docstring states the required behaviour, followed by its body. Does the body correctly implement the docstring for all valid inputs (it would pass a hidden test suite written from the docstring)?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7855,
        "no": 0.2145
       },
       "probability_of_expected": 0.7855,
       "proper_score": 0.954
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.98,
        "no": 0.02
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9996
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.894,
        "no": 0.106
       },
       "probability_of_expected": 0.894,
       "proper_score": 0.9888
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.6086,
        "no": 0.3914
       },
       "probability_of_expected": 0.6086,
       "proper_score": 0.8468
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9997,
        "no": 0.0003
       },
       "probability_of_expected": 0.9997,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.8535,
        "no": 0.1465
       },
       "probability_of_expected": 0.8535,
       "proper_score": 0.9785
      }
     }
    },
    {
     "id": "defect_type",
     "type": "choice",
     "instructions": "Which single kind of defect does the body have relative to its docstring? Pick no_bug if it is correct.",
     "options": [
      [
       "no_bug",
       "The body implements the docstring correctly."
      ],
      [
       "missing_logic",
       "A needed step, condition or case is absent."
      ],
      [
       "excess_logic",
       "An extra step or condition changes the result."
      ],
      [
       "value_misuse",
       "A wrong literal, constant or initial value is used."
      ],
      [
       "operator_misuse",
       "A wrong operator or comparison is used."
      ],
      [
       "variable_misuse",
       "The wrong variable is read or written."
      ],
      [
       "function_misuse",
       "The wrong function or method is called."
      ]
     ],
     "graded": true,
     "expected": "no_bug",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.7464,
        "function_misuse": 0.1091,
        "excess_logic": 0.0506,
        "missing_logic": 0.0398,
        "value_misuse": 0.0271,
        "operator_misuse": 0.0193
       },
       "probability_of_expected": 0.7464,
       "proper_score": 0.9592
      },
      "Jev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 1.0,
        "variable_misuse": 0.0,
        "excess_logic": 0.0,
        "operator_misuse": 0.0,
        "value_misuse": 0.0,
        "function_misuse": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.9892,
        "missing_logic": 0.0034,
        "operator_misuse": 0.0024,
        "excess_logic": 0.0019,
        "function_misuse": 0.0014,
        "variable_misuse": 0.0009
       },
       "probability_of_expected": 0.9892,
       "proper_score": 0.9999
      },
      "NanoJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.3603,
        "excess_logic": 0.1635,
        "missing_logic": 0.1049,
        "value_misuse": 0.1012,
        "operator_misuse": 0.0993,
        "variable_misuse": 0.09
       },
       "probability_of_expected": 0.3603,
       "proper_score": 0.7592
      },
      "OpenJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.9999,
        "missing_logic": 0.0001,
        "value_misuse": 0.0,
        "function_misuse": 0.0,
        "excess_logic": 0.0,
        "variable_misuse": 0.0
       },
       "probability_of_expected": 0.9999,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.9485,
        "missing_logic": 0.0152,
        "excess_logic": 0.0116,
        "operator_misuse": 0.0082,
        "variable_misuse": 0.0076,
        "value_misuse": 0.0063
       },
       "probability_of_expected": 0.9485,
       "proper_score": 0.9984
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 3077.2,
    "Jev": 147.5,
    "JevFish": 2364.6,
    "NanoJev": 619.5,
    "OpenJev": 344.5,
    "System One Scorer": 2996.5
   }
  },
  {
   "id": "safety-and-quality:heval-Python-35-ok",
   "state": {
    "file": "solution.py",
    "function": "def max_element(l: list):\n    \"\"\"Return maximum element in the list.\n    >>> max_element([1, 2, 3])\n    3\n    >>> max_element([5, 3, -5, 2, -3, 3, 9, 0, 123, 1, -10])\n    123\n    \"\"\"\n    m = l[0]\n    for e in l:\n        if e > m:\n            m = e\n    return m"
   },
   "state_tokens": 131,
   "tier": "common",
   "primary": "implements_spec",
   "questions": [
    {
     "id": "implements_spec",
     "type": "noul",
     "instructions": "The state holds a Python function whose docstring states the required behaviour, followed by its body. Does the body correctly implement the docstring for all valid inputs (it would pass a hidden test suite written from the docstring)?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9013,
        "no": 0.0987
       },
       "probability_of_expected": 0.9013,
       "proper_score": 0.9903
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.77,
        "no": 0.23
       },
       "probability_of_expected": 0.77,
       "proper_score": 0.9471
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7636,
        "no": 0.2364
       },
       "probability_of_expected": 0.7636,
       "proper_score": 0.9441
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.5789,
        "no": 0.4211
       },
       "probability_of_expected": 0.5789,
       "proper_score": 0.8227
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9626,
        "no": 0.0374
       },
       "probability_of_expected": 0.9626,
       "proper_score": 0.9986
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.762,
        "no": 0.238
       },
       "probability_of_expected": 0.762,
       "proper_score": 0.9433
      }
     }
    },
    {
     "id": "defect_type",
     "type": "choice",
     "instructions": "Which single kind of defect does the body have relative to its docstring? Pick no_bug if it is correct.",
     "options": [
      [
       "no_bug",
       "The body implements the docstring correctly."
      ],
      [
       "missing_logic",
       "A needed step, condition or case is absent."
      ],
      [
       "excess_logic",
       "An extra step or condition changes the result."
      ],
      [
       "value_misuse",
       "A wrong literal, constant or initial value is used."
      ],
      [
       "operator_misuse",
       "A wrong operator or comparison is used."
      ],
      [
       "variable_misuse",
       "The wrong variable is read or written."
      ],
      [
       "function_misuse",
       "The wrong function or method is called."
      ]
     ],
     "graded": true,
     "expected": "no_bug",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.7633,
        "missing_logic": 0.0627,
        "excess_logic": 0.0597,
        "value_misuse": 0.0407,
        "operator_misuse": 0.0335,
        "variable_misuse": 0.0305
       },
       "probability_of_expected": 0.7633,
       "proper_score": 0.9663
      },
      "Jev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.61,
        "missing_logic": 0.38,
        "value_misuse": 0.01,
        "function_misuse": 0.0,
        "excess_logic": 0.0,
        "variable_misuse": 0.0
       },
       "probability_of_expected": 0.61,
       "proper_score": 0.8517
      },
      "JevFish": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.9403,
        "missing_logic": 0.0206,
        "operator_misuse": 0.0136,
        "excess_logic": 0.009,
        "variable_misuse": 0.0074,
        "value_misuse": 0.0051
       },
       "probability_of_expected": 0.9403,
       "proper_score": 0.9978
      },
      "NanoJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.2446,
        "excess_logic": 0.2014,
        "value_misuse": 0.1208,
        "operator_misuse": 0.117,
        "missing_logic": 0.1134,
        "variable_misuse": 0.1053
       },
       "probability_of_expected": 0.2446,
       "proper_score": 0.6635
      },
      "OpenJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.9867,
        "missing_logic": 0.0083,
        "function_misuse": 0.0018,
        "operator_misuse": 0.0014,
        "variable_misuse": 0.0008,
        "value_misuse": 0.0007
       },
       "probability_of_expected": 0.9867,
       "proper_score": 0.9999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no_bug",
       "correct": true,
       "probabilities": {
        "no_bug": 0.8688,
        "missing_logic": 0.0374,
        "value_misuse": 0.0277,
        "operator_misuse": 0.0227,
        "variable_misuse": 0.0214,
        "excess_logic": 0.0189
       },
       "probability_of_expected": 0.8688,
       "proper_score": 0.9896
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 3017.8,
    "Jev": 185.8,
    "JevFish": 2231.1,
    "NanoJev": 731.8,
    "OpenJev": 362.5,
    "System One Scorer": 3198.9
   }
  },
  {
   "id": "safety-and-quality:heval-Python-60-buggy",
   "state": {
    "file": "solution.py",
    "function": "def sum_to_n(n: int):\n    \"\"\"sum_to_n is a function that sums numbers from 1 to n.\n    >>> sum_to_n(30)\n    465\n    >>> sum_to_n(100)\n    5050\n    >>> sum_to_n(5)\n    15\n    >>> sum_to_n(10)\n    55\n    >>> sum_to_n(1)\n    1\n    \"\"\"\n    return sum(range(n))"
   },
   "state_tokens": 125,
   "tier": "common",
   "primary": "implements_spec",
   "questions": [
    {
     "id": "implements_spec",
     "type": "noul",
     "instructions": "The state holds a Python function whose docstring states the required behaviour, followed by its body. Does the body correctly implement the docstring for all valid inputs (it would pass a hidden test suite written from the docstring)?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.9013,
        "no": 0.0987
       },
       "probability_of_expected": 0.0987,
       "proper_score": 0.1877
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.05,
        "no": 0.95
       },
       "probability_of_expected": 0.95,
       "proper_score": 0.9975
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.7128,
        "no": 0.2872
       },
       "probability_of_expected": 0.2872,
       "proper_score": 0.4919
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.61,
        "no": 0.39
       },
       "probability_of_expected": 0.39,
       "proper_score": 0.6279
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0006,
        "no": 0.9994
       },
       "probability_of_expected": 0.9994,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.826,
        "no": 0.174
       },
       "probability_of_expected": 0.174,
       "proper_score": 0.3177
      }
     }
    },
    {
     "id": "defect_type",
     "type": "choice",
     "instructions": "Which single kind of defect does the body have relative to its docstring? Pick no_bug if it is correct.",
     "options": [
      [
       "no_bug",
       "The body implements the docstring correctly."
      ],
      [
       "missing_logic",
       "A needed step, condition or case is absent."
      ],
      [
       "excess_logic",
       "An extra step or condition changes the result."
      ],
      [
       "value_misuse",
       "A wrong literal, constant or initial value is used."
      ],
      [
       "operator_misuse",
       "A wrong operator or comparison is used."
      ],
      [
       "variable_misuse",
       "The wrong variable is read or written."
      ],
      [
       "function_misuse",
       "The wrong function or method is called."
      ]
     ],
     "graded": true,
     "expected": "value_misuse",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.8615,
        "value_misuse": 0.0397,
        "missing_logic": 0.0379,
        "excess_logic": 0.0298,
        "operator_misuse": 0.0125,
        "function_misuse": 0.0119
       },
       "probability_of_expected": 0.0397,
       "proper_score": 0.1665
      },
      "Jev": {
       "answered": true,
       "pick": "missing_logic",
       "correct": false,
       "probabilities": {
        "missing_logic": 0.7,
        "value_misuse": 0.14,
        "no_bug": 0.07,
        "function_misuse": 0.05,
        "operator_misuse": 0.02,
        "variable_misuse": 0.01
       },
       "probability_of_expected": 0.14,
       "proper_score": 0.3812
      },
      "JevFish": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.9274,
        "missing_logic": 0.0279,
        "excess_logic": 0.0102,
        "function_misuse": 0.0097,
        "operator_misuse": 0.0084,
        "variable_misuse": 0.0083,
        "value_misuse": 0.0082
       },
       "probability_of_expected": 0.0082,
       "proper_score": 0.0775
      },
      "NanoJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.2642,
        "excess_logic": 0.1886,
        "value_misuse": 0.1211,
        "missing_logic": 0.1115,
        "operator_misuse": 0.109,
        "function_misuse": 0.1044
       },
       "probability_of_expected": 0.1211,
       "proper_score": 0.5383
      },
      "OpenJev": {
       "answered": true,
       "pick": "operator_misuse",
       "correct": false,
       "probabilities": {
        "operator_misuse": 0.874,
        "missing_logic": 0.0872,
        "excess_logic": 0.0134,
        "value_misuse": 0.0099,
        "variable_misuse": 0.0057,
        "no_bug": 0.0051
       },
       "probability_of_expected": 0.0099,
       "proper_score": 0.1239
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.9066,
        "missing_logic": 0.0305,
        "excess_logic": 0.0168,
        "operator_misuse": 0.0153,
        "value_misuse": 0.0137,
        "variable_misuse": 0.0137
       },
       "probability_of_expected": 0.0137,
       "proper_score": 0.1018
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 3745.9,
    "Jev": 185.1,
    "JevFish": 2410.2,
    "NanoJev": 698.5,
    "OpenJev": 409.0,
    "System One Scorer": 3168.6
   }
  },
  {
   "id": "safety-and-quality:heval-Python-61-buggy",
   "state": {
    "file": "solution.py",
    "function": "def correct_bracketing(brackets: str):\n    \"\"\" brackets is a string of \"(\" and \")\".\n    return True if every opening bracket has a corresponding closing bracket.\n\n    >>> correct_bracketing(\"(\")\n    False\n    >>> correct_bracketing(\"()\")\n    True\n    >>> correct_bracketing(\"(()())\")\n    True\n    >>> correct_bracketing(\")(()\")\n    False\n    \"\"\"\n    depth = 0\n    for b in brackets:\n        if b == \"(\":\n            depth += 1\n        else:\n            depth -= 1\n        if depth < 0:\n            return True\n    return depth == 0"
   },
   "state_tokens": 171,
   "tier": "common",
   "primary": "implements_spec",
   "questions": [
    {
     "id": "implements_spec",
     "type": "noul",
     "instructions": "The state holds a Python function whose docstring states the required behaviour, followed by its body. Does the body correctly implement the docstring for all valid inputs (it would pass a hidden test suite written from the docstring)?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.6514,
        "no": 0.3486
       },
       "probability_of_expected": 0.3486,
       "proper_score": 0.5757
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.03,
        "no": 0.97
       },
       "probability_of_expected": 0.97,
       "proper_score": 0.9991
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.741,
        "no": 0.259
       },
       "probability_of_expected": 0.259,
       "proper_score": 0.4508
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.6406,
        "no": 0.3594
       },
       "probability_of_expected": 0.3594,
       "proper_score": 0.5896
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0003,
        "no": 0.9997
       },
       "probability_of_expected": 0.9997,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.5,
        "no": 0.5
       },
       "probability_of_expected": 0.5,
       "proper_score": 0.75
      }
     }
    },
    {
     "id": "defect_type",
     "type": "choice",
     "instructions": "Which single kind of defect does the body have relative to its docstring? Pick no_bug if it is correct.",
     "options": [
      [
       "no_bug",
       "The body implements the docstring correctly."
      ],
      [
       "missing_logic",
       "A needed step, condition or case is absent."
      ],
      [
       "excess_logic",
       "An extra step or condition changes the result."
      ],
      [
       "value_misuse",
       "A wrong literal, constant or initial value is used."
      ],
      [
       "operator_misuse",
       "A wrong operator or comparison is used."
      ],
      [
       "variable_misuse",
       "The wrong variable is read or written."
      ],
      [
       "function_misuse",
       "The wrong function or method is called."
      ]
     ],
     "graded": true,
     "expected": "operator_misuse",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.4753,
        "excess_logic": 0.2202,
        "missing_logic": 0.1499,
        "operator_misuse": 0.0883,
        "value_misuse": 0.0279,
        "variable_misuse": 0.0266
       },
       "probability_of_expected": 0.0883,
       "proper_score": 0.4351
      },
      "Jev": {
       "answered": true,
       "pick": "value_misuse",
       "correct": false,
       "probabilities": {
        "value_misuse": 0.47,
        "excess_logic": 0.26,
        "missing_logic": 0.13,
        "operator_misuse": 0.13,
        "variable_misuse": 0.01,
        "no_bug": 0.0
       },
       "probability_of_expected": 0.13,
       "proper_score": 0.4688
      },
      "JevFish": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.9396,
        "missing_logic": 0.0283,
        "excess_logic": 0.0116,
        "operator_misuse": 0.0085,
        "variable_misuse": 0.0051,
        "function_misuse": 0.004
       },
       "probability_of_expected": 0.0085,
       "proper_score": 0.0666
      },
      "NanoJev": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.429,
        "excess_logic": 0.1464,
        "missing_logic": 0.0954,
        "value_misuse": 0.0889,
        "operator_misuse": 0.0845,
        "variable_misuse": 0.0816
       },
       "probability_of_expected": 0.0845,
       "proper_score": 0.4636
      },
      "OpenJev": {
       "answered": true,
       "pick": "operator_misuse",
       "correct": true,
       "probabilities": {
        "operator_misuse": 0.9169,
        "no_bug": 0.0279,
        "excess_logic": 0.0175,
        "missing_logic": 0.0132,
        "value_misuse": 0.0107,
        "function_misuse": 0.0097
       },
       "probability_of_expected": 0.9169,
       "proper_score": 0.9958
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no_bug",
       "correct": false,
       "probabilities": {
        "no_bug": 0.5624,
        "missing_logic": 0.1688,
        "excess_logic": 0.1556,
        "operator_misuse": 0.0444,
        "variable_misuse": 0.0333,
        "value_misuse": 0.0288
       },
       "probability_of_expected": 0.0444,
       "proper_score": 0.3579
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 3714.6,
    "Jev": 164.2,
    "JevFish": 2398.9,
    "NanoJev": 713.2,
    "OpenJev": 402.8,
    "System One Scorer": 3173.5
   }
  }
 ]
}