{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "arxiv-archive-subcategory",
 "family": "data-and-operations",
 "use_case": "hierarchical-classification",
 "vertical": "science-publishing",
 "dataset": "gfissore/arxiv-abstracts-2021",
 "licence": "CC0-1.0",
 "provenance": "outcome",
 "state_kind": "text",
 "primitive": "choice",
 "items": 200,
 "scores": {
  "decider-2b": {
   "decisionScore": 70.09,
   "accuracy": 0.78,
   "answered": 200,
   "items": 200
  },
  "Jev": {
   "decisionScore": 76.85,
   "accuracy": 0.885,
   "answered": 200,
   "items": 200
  },
  "JevFish": {
   "decisionScore": 69.31,
   "accuracy": 0.785,
   "answered": 200,
   "items": 200
  },
  "NanoJev": {
   "decisionScore": 53.67,
   "accuracy": 0.295,
   "answered": 200,
   "items": 200
  },
  "OpenJev": {
   "decisionScore": 65.41,
   "accuracy": 0.865,
   "answered": 200,
   "items": 200
  },
  "System One Scorer": {
   "decisionScore": 70.57,
   "accuracy": 0.815,
   "answered": 200,
   "items": 200
  }
 },
 "samples": [
  {
   "id": "data-and-operations:arxiv-34d4b5d025fe",
   "state": {
    "title": "New examples of Calabi-Yau threefolds and genus zero surfaces",
    "abstract": "We classify the subgroups of the automorphism group of the product of 4 projective lines admitting an invariant anticanonical smooth divisor on which the action is free. As a first application, we describe new examples of Calabi-Yau 3-folds with small Hodge numbers. In particular, the Picard number is 1 and the number of moduli is 5. Furthermore, the fundamental group is non-trivial. We also construct a new family of minimal surfaces of general type with geometric genus zero, K^2=3 and fundamental group of order 16. We show that this family dominates an irreducible component of dimension 4 of the moduli space of the surfaces of general type."
   },
   "state_tokens": 166,
   "tier": "common",
   "primary": "archive",
   "questions": [
    {
     "id": "archive",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Which arXiv archive (top-level subject group) is this paper's primary category in?",
     "options": [
      [
       "astro_ph",
       "Astrophysics"
      ],
      [
       "cond_mat",
       "Condensed matter physics"
      ],
      [
       "cs",
       "Computer science"
      ],
      [
       "econ",
       "Economics"
      ],
      [
       "eess",
       "Electrical engineering and systems science"
      ],
      [
       "math",
       "Mathematics"
      ],
      [
       "nlin",
       "Nonlinear sciences"
      ],
      [
       "physics",
       "Physics (general physics archive: optics, fluids, plasma, atomic, chemical, applied, ...)"
      ],
      [
       "q_bio",
       "Quantitative biology"
      ],
      [
       "q_fin",
       "Quantitative finance"
      ],
      [
       "stat",
       "Statistics"
      ]
     ],
     "graded": true,
     "expected": "math",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.9899,
        "nlin": 0.0034,
        "astro_ph": 0.0021,
        "physics": 0.0018,
        "cs": 0.0008,
        "eess": 0.0005
       },
       "probability_of_expected": 0.9899,
       "proper_score": 0.9999
      },
      "Jev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 1.0,
        "q_fin": 0.0,
        "stat": 0.0,
        "cs": 0.0,
        "astro_ph": 0.0,
        "cond_mat": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.9674,
        "astro_ph": 0.0077,
        "stat": 0.007,
        "nlin": 0.0038,
        "eess": 0.003,
        "physics": 0.0027
       },
       "probability_of_expected": 0.9674,
       "proper_score": 0.9994
      },
      "NanoJev": {
       "answered": true,
       "pick": "stat",
       "correct": false,
       "probabilities": {
        "stat": 0.428,
        "math": 0.3353,
        "physics": 0.0672,
        "nlin": 0.0456,
        "cs": 0.0267,
        "astro_ph": 0.0218
       },
       "probability_of_expected": 0.3353,
       "proper_score": 0.683
      },
      "OpenJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.9993,
        "nlin": 0.0003,
        "stat": 0.0002,
        "astro_ph": 0.0001,
        "eess": 0.0001,
        "econ": 0.0
       },
       "probability_of_expected": 0.9993,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.7132,
        "econ": 0.0597,
        "q_fin": 0.0395,
        "nlin": 0.0307,
        "astro_ph": 0.0291,
        "cond_mat": 0.0273
       },
       "probability_of_expected": 0.7132,
       "proper_score": 0.9539
      }
     }
    },
    {
     "id": "subcategory",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Within the paper's true archive, which subcategory is its primary category? The options are that archive's subcategories.",
     "options": [
      [
       "math_ac",
       "math.ac"
      ],
      [
       "math_ag",
       "math.ag"
      ],
      [
       "math_ap",
       "math.ap"
      ],
      [
       "math_at",
       "math.at"
      ],
      [
       "math_ca",
       "math.ca"
      ],
      [
       "math_co",
       "math.co"
      ],
      [
       "math_ct",
       "math.ct"
      ],
      [
       "math_cv",
       "math.cv"
      ],
      [
       "math_dg",
       "math.dg"
      ],
      [
       "math_ds",
       "math.ds"
      ],
      [
       "math_fa",
       "math.fa"
      ],
      [
       "math_gm",
       "math.gm"
      ],
      [
       "math_gn",
       "math.gn"
      ],
      [
       "math_gr",
       "math.gr"
      ],
      [
       "math_gt",
       "math.gt"
      ],
      [
       "math_ho",
       "math.ho"
      ],
      [
       "math_kt",
       "math.kt"
      ],
      [
       "math_lo",
       "math.lo"
      ],
      [
       "math_mg",
       "math.mg"
      ],
      [
       "math_na",
       "math.na"
      ],
      [
       "math_nt",
       "math.nt"
      ],
      [
       "math_oa",
       "math.oa"
      ],
      [
       "math_oc",
       "math.oc"
      ],
      [
       "math_pr",
       "math.pr"
      ],
      [
       "math_qa",
       "math.qa"
      ],
      [
       "math_ra",
       "math.ra"
      ],
      [
       "math_rt",
       "math.rt"
      ],
      [
       "math_sg",
       "math.sg"
      ],
      [
       "math_sp",
       "math.sp"
      ],
      [
       "math_st",
       "math.st"
      ]
     ],
     "graded": true,
     "expected": "math_ag",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.2933,
        "math_dg": 0.1996,
        "math_ag": 0.1234,
        "math_ca": 0.084,
        "math_ap": 0.0389,
        "math_co": 0.0321
       },
       "probability_of_expected": 0.1234,
       "proper_score": 0.5464
      },
      "Jev": {
       "answered": true,
       "pick": "math_ag",
       "correct": true,
       "probabilities": {
        "math_ag": 0.96,
        "math_cv": 0.02,
        "math_ca": 0.01,
        "math_dg": 0.01,
        "math_lo": 0.0,
        "math_sp": 0.0
       },
       "probability_of_expected": 0.96,
       "proper_score": 0.9989
      },
      "JevFish": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.1984,
        "math_st": 0.1764,
        "math_sg": 0.0657,
        "math_dg": 0.0619,
        "math_ag": 0.0509,
        "math_sp": 0.0421
       },
       "probability_of_expected": 0.0509,
       "proper_score": 0.5055
      },
      "NanoJev": {
       "answered": true,
       "pick": "math_qa",
       "correct": false,
       "probabilities": {
        "math_qa": 0.0492,
        "math_dg": 0.0481,
        "math_mg": 0.0469,
        "math_gn": 0.0468,
        "math_gm": 0.0455,
        "math_na": 0.0428,
        "math_ag": 0.035
       },
       "probability_of_expected": 0.035,
       "proper_score": 0.5171
      },
      "OpenJev": {
       "answered": true,
       "pick": "math_ca",
       "correct": false,
       "probabilities": {
        "math_ca": 0.3432,
        "math_ct": 0.1643,
        "math_gm": 0.0929,
        "math_st": 0.0858,
        "math_sp": 0.0731,
        "math_ap": 0.0443,
        "math_ag": 0.0109
       },
       "probability_of_expected": 0.0109,
       "proper_score": 0.4245
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math_ag",
       "correct": true,
       "probabilities": {
        "math_ag": 0.1382,
        "math_mg": 0.1382,
        "math_ac": 0.0678,
        "math_ct": 0.061,
        "math_oa": 0.0548,
        "math_at": 0.0545
       },
       "probability_of_expected": 0.1382,
       "proper_score": 0.6045
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 5430.2,
    "Jev": 255.4,
    "JevFish": 2664.0,
    "NanoJev": 2315.1,
    "OpenJev": 424.8,
    "System One Scorer": 13291.4
   }
  },
  {
   "id": "data-and-operations:arxiv-623ca84587da",
   "state": {
    "title": "On the existence of certain weak Fano threefolds of Picard number two",
    "abstract": "This article settles the question of existence of smooth weak Fano threefolds of Picard number two with small anti-canonical map and previously classified numerical invariants obtained by blowing up certain curves on smooth Fano threefolds of Picard number 1 with the exception of 12 numerical cases."
   },
   "state_tokens": 84,
   "tier": "common",
   "primary": "archive",
   "questions": [
    {
     "id": "archive",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Which arXiv archive (top-level subject group) is this paper's primary category in?",
     "options": [
      [
       "astro_ph",
       "Astrophysics"
      ],
      [
       "cond_mat",
       "Condensed matter physics"
      ],
      [
       "cs",
       "Computer science"
      ],
      [
       "econ",
       "Economics"
      ],
      [
       "eess",
       "Electrical engineering and systems science"
      ],
      [
       "math",
       "Mathematics"
      ],
      [
       "nlin",
       "Nonlinear sciences"
      ],
      [
       "physics",
       "Physics (general physics archive: optics, fluids, plasma, atomic, chemical, applied, ...)"
      ],
      [
       "q_bio",
       "Quantitative biology"
      ],
      [
       "q_fin",
       "Quantitative finance"
      ],
      [
       "stat",
       "Statistics"
      ]
     ],
     "graded": true,
     "expected": "math",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.993,
        "nlin": 0.0027,
        "physics": 0.0012,
        "astro_ph": 0.0006,
        "cs": 0.0005,
        "cond_mat": 0.0004
       },
       "probability_of_expected": 0.993,
       "proper_score": 1.0
      },
      "Jev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 1.0,
        "physics": 0.0,
        "nlin": 0.0,
        "astro_ph": 0.0,
        "cs": 0.0,
        "stat": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.9619,
        "stat": 0.009,
        "astro_ph": 0.009,
        "nlin": 0.0043,
        "eess": 0.004,
        "physics": 0.0026
       },
       "probability_of_expected": 0.9619,
       "proper_score": 0.9992
      },
      "NanoJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.4754,
        "stat": 0.321,
        "physics": 0.1067,
        "cs": 0.0183,
        "nlin": 0.0181,
        "econ": 0.0154
       },
       "probability_of_expected": 0.4754,
       "proper_score": 0.8045
      },
      "OpenJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.9991,
        "nlin": 0.0002,
        "astro_ph": 0.0002,
        "cond_mat": 0.0001,
        "eess": 0.0001,
        "stat": 0.0001
       },
       "probability_of_expected": 0.9991,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.6475,
        "econ": 0.0944,
        "q_fin": 0.0554,
        "stat": 0.0362,
        "astro_ph": 0.0317,
        "physics": 0.0285
       },
       "probability_of_expected": 0.6475,
       "proper_score": 0.9291
      }
     }
    },
    {
     "id": "subcategory",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Within the paper's true archive, which subcategory is its primary category? The options are that archive's subcategories.",
     "options": [
      [
       "math_ac",
       "math.ac"
      ],
      [
       "math_ag",
       "math.ag"
      ],
      [
       "math_ap",
       "math.ap"
      ],
      [
       "math_at",
       "math.at"
      ],
      [
       "math_ca",
       "math.ca"
      ],
      [
       "math_co",
       "math.co"
      ],
      [
       "math_ct",
       "math.ct"
      ],
      [
       "math_cv",
       "math.cv"
      ],
      [
       "math_dg",
       "math.dg"
      ],
      [
       "math_ds",
       "math.ds"
      ],
      [
       "math_fa",
       "math.fa"
      ],
      [
       "math_gm",
       "math.gm"
      ],
      [
       "math_gn",
       "math.gn"
      ],
      [
       "math_gr",
       "math.gr"
      ],
      [
       "math_gt",
       "math.gt"
      ],
      [
       "math_ho",
       "math.ho"
      ],
      [
       "math_kt",
       "math.kt"
      ],
      [
       "math_lo",
       "math.lo"
      ],
      [
       "math_mg",
       "math.mg"
      ],
      [
       "math_na",
       "math.na"
      ],
      [
       "math_nt",
       "math.nt"
      ],
      [
       "math_oa",
       "math.oa"
      ],
      [
       "math_oc",
       "math.oc"
      ],
      [
       "math_pr",
       "math.pr"
      ],
      [
       "math_qa",
       "math.qa"
      ],
      [
       "math_ra",
       "math.ra"
      ],
      [
       "math_rt",
       "math.rt"
      ],
      [
       "math_sg",
       "math.sg"
      ],
      [
       "math_sp",
       "math.sp"
      ],
      [
       "math_st",
       "math.st"
      ]
     ],
     "graded": true,
     "expected": "math_ag",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.341,
        "math_dg": 0.201,
        "math_ag": 0.1435,
        "math_ca": 0.0411,
        "math_ap": 0.0374,
        "math_at": 0.028
       },
       "probability_of_expected": 0.1435,
       "proper_score": 0.5516
      },
      "Jev": {
       "answered": true,
       "pick": "math_ag",
       "correct": true,
       "probabilities": {
        "math_ag": 0.98,
        "math_cv": 0.01,
        "math_ac": 0.01,
        "math_gm": 0.0,
        "math_na": 0.0,
        "math_co": 0.0
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9997
      },
      "JevFish": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.206,
        "math_st": 0.1774,
        "math_ag": 0.0637,
        "math_fa": 0.0574,
        "math_sg": 0.0473,
        "math_sp": 0.0425
       },
       "probability_of_expected": 0.0637,
       "proper_score": 0.5172
      },
      "NanoJev": {
       "answered": true,
       "pick": "math_qa",
       "correct": false,
       "probabilities": {
        "math_qa": 0.0609,
        "math_gn": 0.0579,
        "math_gm": 0.0538,
        "math_mg": 0.0491,
        "math_dg": 0.0488,
        "math_oa": 0.0445,
        "math_ag": 0.034
       },
       "probability_of_expected": 0.034,
       "proper_score": 0.5153
      },
      "OpenJev": {
       "answered": true,
       "pick": "math_st",
       "correct": false,
       "probabilities": {
        "math_st": 0.2908,
        "math_ca": 0.247,
        "math_at": 0.0682,
        "math_ct": 0.0568,
        "math_ap": 0.0492,
        "math_sp": 0.0487,
        "math_ag": 0.0123
       },
       "probability_of_expected": 0.0123,
       "proper_score": 0.43
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math_mg",
       "correct": false,
       "probabilities": {
        "math_mg": 0.1426,
        "math_ag": 0.105,
        "math_fa": 0.0738,
        "math_ac": 0.0668,
        "math_ct": 0.0613,
        "math_cv": 0.0601
       },
       "probability_of_expected": 0.105,
       "proper_score": 0.5732
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4771.3,
    "Jev": 192.1,
    "JevFish": 2392.5,
    "NanoJev": 1668.3,
    "OpenJev": 394.9,
    "System One Scorer": 8963.3
   }
  },
  {
   "id": "data-and-operations:arxiv-896e849c9b4e",
   "state": {
    "title": "Stochastic theory of protein synthesis and polysome: ribosome profile on a single mRNA transcript",
    "abstract": "The process of polymerizing a protein by a ribosome, using a messenger RNA (mRNA) as the corresponding template, is called {\\it translation}. Ribosome may be regarded as a molecular motor for which the mRNA template serves also as the track. Often several ribosomes may translate the same (mRNA) simultaneously. The ribosomes bound simultaneously to a single mRNA transcript are the members of a polyribosome (or, simply, {\\it polysome}). Experimentally measured {\\it polysome profile} gives the distribution of polysome {\\it sizes}. Recently a breakthrough in determining the instantaneous {\\it positions} of the ribosomes on a given mRNA track has been achieved and the technique is called {\\it ribosome profiling} \\cite{ingolia10,guo10}."
   },
   "state_tokens": 191,
   "tier": "common",
   "primary": "archive",
   "questions": [
    {
     "id": "archive",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Which arXiv archive (top-level subject group) is this paper's primary category in?",
     "options": [
      [
       "astro_ph",
       "Astrophysics"
      ],
      [
       "cond_mat",
       "Condensed matter physics"
      ],
      [
       "cs",
       "Computer science"
      ],
      [
       "econ",
       "Economics"
      ],
      [
       "eess",
       "Electrical engineering and systems science"
      ],
      [
       "math",
       "Mathematics"
      ],
      [
       "nlin",
       "Nonlinear sciences"
      ],
      [
       "physics",
       "Physics (general physics archive: optics, fluids, plasma, atomic, chemical, applied, ...)"
      ],
      [
       "q_bio",
       "Quantitative biology"
      ],
      [
       "q_fin",
       "Quantitative finance"
      ],
      [
       "stat",
       "Statistics"
      ]
     ],
     "graded": true,
     "expected": "physics",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "q_bio",
       "correct": false,
       "probabilities": {
        "q_bio": 0.8826,
        "math": 0.0305,
        "stat": 0.0229,
        "cond_mat": 0.0189,
        "nlin": 0.0148,
        "physics": 0.0141
       },
       "probability_of_expected": 0.0141,
       "proper_score": 0.1234
      },
      "Jev": {
       "answered": true,
       "pick": "q_bio",
       "correct": false,
       "probabilities": {
        "q_bio": 0.97,
        "physics": 0.02,
        "math": 0.01,
        "astro_ph": 0.0,
        "cs": 0.0,
        "stat": 0.0
       },
       "probability_of_expected": 0.02,
       "proper_score": 0.0493
      },
      "JevFish": {
       "answered": true,
       "pick": "q_bio",
       "correct": false,
       "probabilities": {
        "q_bio": 0.8182,
        "nlin": 0.0469,
        "cond_mat": 0.0293,
        "math": 0.0209,
        "physics": 0.0195,
        "stat": 0.0164
       },
       "probability_of_expected": 0.0195,
       "proper_score": 0.1824
      },
      "NanoJev": {
       "answered": true,
       "pick": "stat",
       "correct": false,
       "probabilities": {
        "stat": 0.7217,
        "q_bio": 0.1648,
        "physics": 0.0751,
        "math": 0.0183,
        "nlin": 0.0067,
        "astro_ph": 0.0039
       },
       "probability_of_expected": 0.0751,
       "proper_score": 0.2981
      },
      "OpenJev": {
       "answered": true,
       "pick": "q_bio",
       "correct": false,
       "probabilities": {
        "q_bio": 0.6124,
        "physics": 0.2771,
        "astro_ph": 0.0585,
        "cond_mat": 0.0454,
        "eess": 0.0031,
        "cs": 0.0014
       },
       "probability_of_expected": 0.2771,
       "proper_score": 0.5484
      },
      "System One Scorer": {
       "answered": true,
       "pick": "q_bio",
       "correct": false,
       "probabilities": {
        "q_bio": 0.7689,
        "math": 0.0587,
        "stat": 0.0481,
        "econ": 0.0256,
        "nlin": 0.0218,
        "cond_mat": 0.0174,
        "physics": 0.0169
       },
       "probability_of_expected": 0.0169,
       "proper_score": 0.2173
      }
     }
    },
    {
     "id": "subcategory",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Within the paper's true archive, which subcategory is its primary category? The options are that archive's subcategories.",
     "options": [
      [
       "physics_acc_ph",
       "physics.acc.ph"
      ],
      [
       "physics_ao_ph",
       "physics.ao.ph"
      ],
      [
       "physics_atm_clus",
       "physics.atm.clus"
      ],
      [
       "physics_atom_ph",
       "physics.atom.ph"
      ],
      [
       "physics_bio_ph",
       "physics.bio.ph"
      ],
      [
       "physics_chem_ph",
       "physics.chem.ph"
      ],
      [
       "physics_class_ph",
       "physics.class.ph"
      ],
      [
       "physics_comp_ph",
       "physics.comp.ph"
      ],
      [
       "physics_data_an",
       "physics.data.an"
      ],
      [
       "physics_ed_ph",
       "physics.ed.ph"
      ],
      [
       "physics_flu_dyn",
       "physics.flu.dyn"
      ],
      [
       "physics_gen_ph",
       "physics.gen.ph"
      ],
      [
       "physics_geo_ph",
       "physics.geo.ph"
      ],
      [
       "physics_hist_ph",
       "physics.hist.ph"
      ],
      [
       "physics_ins_det",
       "physics.ins.det"
      ],
      [
       "physics_med_ph",
       "physics.med.ph"
      ],
      [
       "physics_optics",
       "physics.optics"
      ],
      [
       "physics_plasm_ph",
       "physics.plasm.ph"
      ],
      [
       "physics_pop_ph",
       "physics.pop.ph"
      ],
      [
       "physics_soc_ph",
       "physics.soc.ph"
      ],
      [
       "physics_space_ph",
       "physics.space.ph"
      ]
     ],
     "graded": true,
     "expected": "physics_bio_ph",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "physics_bio_ph",
       "correct": true,
       "probabilities": {
        "physics_bio_ph": 0.7261,
        "physics_chem_ph": 0.1416,
        "physics_comp_ph": 0.0251,
        "physics_ao_ph": 0.0228,
        "physics_acc_ph": 0.0106,
        "physics_gen_ph": 0.0091
       },
       "probability_of_expected": 0.7261,
       "proper_score": 0.9516
      },
      "Jev": {
       "answered": true,
       "pick": "physics_bio_ph",
       "correct": true,
       "probabilities": {
        "physics_bio_ph": 0.99,
        "physics_gen_ph": 0.01,
        "physics_hist_ph": 0.0,
        "physics_pop_ph": 0.0,
        "physics_ed_ph": 0.0,
        "physics_class_ph": 0.0
       },
       "probability_of_expected": 0.99,
       "proper_score": 0.9999
      },
      "JevFish": {
       "answered": true,
       "pick": "physics_bio_ph",
       "correct": true,
       "probabilities": {
        "physics_bio_ph": 0.7166,
        "physics_acc_ph": 0.0765,
        "physics_chem_ph": 0.0387,
        "physics_gen_ph": 0.0282,
        "physics_comp_ph": 0.0192,
        "physics_med_ph": 0.0146
       },
       "probability_of_expected": 0.7166,
       "proper_score": 0.955
      },
      "NanoJev": {
       "answered": true,
       "pick": "physics_data_an",
       "correct": false,
       "probabilities": {
        "physics_data_an": 0.0727,
        "physics_hist_ph": 0.0704,
        "physics_flu_dyn": 0.0699,
        "physics_ao_ph": 0.0685,
        "physics_gen_ph": 0.06,
        "physics_acc_ph": 0.0559,
        "physics_bio_ph": 0.0509
       },
       "probability_of_expected": 0.0509,
       "proper_score": 0.525
      },
      "OpenJev": {
       "answered": true,
       "pick": "physics_bio_ph",
       "correct": true,
       "probabilities": {
        "physics_bio_ph": 0.4186,
        "physics_gen_ph": 0.2576,
        "physics_class_ph": 0.0768,
        "physics_acc_ph": 0.0716,
        "physics_atom_ph": 0.0491,
        "physics_ed_ph": 0.0402
       },
       "probability_of_expected": 0.4186,
       "proper_score": 0.7896
      },
      "System One Scorer": {
       "answered": true,
       "pick": "physics_chem_ph",
       "correct": false,
       "probabilities": {
        "physics_chem_ph": 0.1417,
        "physics_soc_ph": 0.1208,
        "physics_ao_ph": 0.1108,
        "physics_comp_ph": 0.101,
        "physics_bio_ph": 0.0617,
        "physics_med_ph": 0.058
       },
       "probability_of_expected": 0.0617,
       "proper_score": 0.522
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 5190.1,
    "Jev": 187.0,
    "JevFish": 2426.3,
    "NanoJev": 1671.6,
    "OpenJev": 554.7,
    "System One Scorer": 9134.7
   }
  },
  {
   "id": "data-and-operations:arxiv-8d6972262680",
   "state": {
    "title": "On the lack of dispersion for a class of magnetic Dirac flows",
    "abstract": "We show that global Strichartz estimates for magnetic Dirac operators generally fails, if the potentials do not decay fast enough at infinity. In order to prove this, we construct some explicit examples of homogeneous magnetic potentials with less than Coulomb decay, i. e. with homogeneity-degree more than -1, such that the magnetic field points to a fixed direction, which does not depend on $x\\in \\mathbb{R}^3$."
   },
   "state_tokens": 112,
   "tier": "common",
   "primary": "archive",
   "questions": [
    {
     "id": "archive",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Which arXiv archive (top-level subject group) is this paper's primary category in?",
     "options": [
      [
       "astro_ph",
       "Astrophysics"
      ],
      [
       "cond_mat",
       "Condensed matter physics"
      ],
      [
       "cs",
       "Computer science"
      ],
      [
       "econ",
       "Economics"
      ],
      [
       "eess",
       "Electrical engineering and systems science"
      ],
      [
       "math",
       "Mathematics"
      ],
      [
       "nlin",
       "Nonlinear sciences"
      ],
      [
       "physics",
       "Physics (general physics archive: optics, fluids, plasma, atomic, chemical, applied, ...)"
      ],
      [
       "q_bio",
       "Quantitative biology"
      ],
      [
       "q_fin",
       "Quantitative finance"
      ],
      [
       "stat",
       "Statistics"
      ]
     ],
     "graded": true,
     "expected": "math",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.7223,
        "nlin": 0.207,
        "physics": 0.0385,
        "cond_mat": 0.017,
        "astro_ph": 0.0044,
        "eess": 0.0025
       },
       "probability_of_expected": 0.7223,
       "proper_score": 0.9391
      },
      "Jev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.95,
        "physics": 0.05,
        "astro_ph": 0.0,
        "nlin": 0.0,
        "econ": 0.0,
        "q_bio": 0.0
       },
       "probability_of_expected": 0.95,
       "proper_score": 0.9975
      },
      "JevFish": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.5677,
        "nlin": 0.3542,
        "physics": 0.0228,
        "cond_mat": 0.0166,
        "stat": 0.0117,
        "astro_ph": 0.0106
       },
       "probability_of_expected": 0.5677,
       "proper_score": 0.8432
      },
      "NanoJev": {
       "answered": true,
       "pick": "stat",
       "correct": false,
       "probabilities": {
        "stat": 0.6335,
        "math": 0.2844,
        "physics": 0.0537,
        "nlin": 0.0135,
        "cond_mat": 0.0041,
        "astro_ph": 0.0035
       },
       "probability_of_expected": 0.2844,
       "proper_score": 0.5418
      },
      "OpenJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.7719,
        "physics": 0.2111,
        "cond_mat": 0.0059,
        "astro_ph": 0.0052,
        "nlin": 0.002,
        "eess": 0.0016
       },
       "probability_of_expected": 0.7719,
       "proper_score": 0.9517
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.5113,
        "cond_mat": 0.1063,
        "econ": 0.0723,
        "nlin": 0.0685,
        "physics": 0.0561,
        "astro_ph": 0.0532
       },
       "probability_of_expected": 0.5113,
       "proper_score": 0.8651
      }
     }
    },
    {
     "id": "subcategory",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Within the paper's true archive, which subcategory is its primary category? The options are that archive's subcategories.",
     "options": [
      [
       "math_ac",
       "math.ac"
      ],
      [
       "math_ag",
       "math.ag"
      ],
      [
       "math_ap",
       "math.ap"
      ],
      [
       "math_at",
       "math.at"
      ],
      [
       "math_ca",
       "math.ca"
      ],
      [
       "math_co",
       "math.co"
      ],
      [
       "math_ct",
       "math.ct"
      ],
      [
       "math_cv",
       "math.cv"
      ],
      [
       "math_dg",
       "math.dg"
      ],
      [
       "math_ds",
       "math.ds"
      ],
      [
       "math_fa",
       "math.fa"
      ],
      [
       "math_gm",
       "math.gm"
      ],
      [
       "math_gn",
       "math.gn"
      ],
      [
       "math_gr",
       "math.gr"
      ],
      [
       "math_gt",
       "math.gt"
      ],
      [
       "math_ho",
       "math.ho"
      ],
      [
       "math_kt",
       "math.kt"
      ],
      [
       "math_lo",
       "math.lo"
      ],
      [
       "math_mg",
       "math.mg"
      ],
      [
       "math_na",
       "math.na"
      ],
      [
       "math_nt",
       "math.nt"
      ],
      [
       "math_oa",
       "math.oa"
      ],
      [
       "math_oc",
       "math.oc"
      ],
      [
       "math_pr",
       "math.pr"
      ],
      [
       "math_qa",
       "math.qa"
      ],
      [
       "math_ra",
       "math.ra"
      ],
      [
       "math_rt",
       "math.rt"
      ],
      [
       "math_sg",
       "math.sg"
      ],
      [
       "math_sp",
       "math.sp"
      ],
      [
       "math_st",
       "math.st"
      ]
     ],
     "graded": true,
     "expected": "math_ap",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.2964,
        "math_ag": 0.1373,
        "math_dg": 0.0701,
        "math_ca": 0.0606,
        "math_ap": 0.0578,
        "math_at": 0.05
       },
       "probability_of_expected": 0.0578,
       "proper_score": 0.4941
      },
      "Jev": {
       "answered": true,
       "pick": "math_fa",
       "correct": false,
       "probabilities": {
        "math_fa": 0.29,
        "math_ac": 0.22,
        "math_sp": 0.11,
        "math_ap": 0.1,
        "math_ds": 0.09,
        "math_oc": 0.07
       },
       "probability_of_expected": 0.1,
       "proper_score": 0.5154
      },
      "JevFish": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.1951,
        "math_st": 0.1845,
        "math_ag": 0.061,
        "math_sg": 0.054,
        "math_sp": 0.0521,
        "math_ap": 0.046
       },
       "probability_of_expected": 0.046,
       "proper_score": 0.5004
      },
      "NanoJev": {
       "answered": true,
       "pick": "math_mg",
       "correct": false,
       "probabilities": {
        "math_mg": 0.0638,
        "math_gm": 0.0588,
        "math_gn": 0.0548,
        "math_qa": 0.0493,
        "math_dg": 0.0474,
        "math_na": 0.0467,
        "math_ap": 0.0253
       },
       "probability_of_expected": 0.0253,
       "proper_score": 0.5064
      },
      "OpenJev": {
       "answered": true,
       "pick": "math_st",
       "correct": false,
       "probabilities": {
        "math_st": 0.3413,
        "math_gm": 0.1101,
        "math_ca": 0.0792,
        "math_ct": 0.0641,
        "math_co": 0.0473,
        "math_ds": 0.0442,
        "math_ap": 0.0304
       },
       "probability_of_expected": 0.0304,
       "proper_score": 0.4555
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math_ap",
       "correct": true,
       "probabilities": {
        "math_ap": 0.0947,
        "math_pr": 0.0812,
        "math_ac": 0.0683,
        "math_ag": 0.0635,
        "math_sp": 0.0545,
        "math_ct": 0.051
       },
       "probability_of_expected": 0.0947,
       "proper_score": 0.5701
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4170.8,
    "Jev": 231.8,
    "JevFish": 2473.1,
    "NanoJev": 1939.0,
    "OpenJev": 414.2,
    "System One Scorer": 10760.8
   }
  },
  {
   "id": "data-and-operations:arxiv-b87c1c2f175e",
   "state": {
    "title": "Tanaka's equation on the circle and stochastic flows",
    "abstract": "We define a Tanaka's equation on an oriented graph with two edges and two vertices. This graph will be embedded in the unit circle. Extending this equation to flows of kernels, we show that the laws of the flows of kernels $K$ solution of Tanaka's equation can be classified by pairs of probability measures $(m^+,m^-)$ on $[0,1]$, with mean 1/2. What happens at the first vertex is governed by $m^+$, and at the second by $m^-$. For each vertex $P$, we construct a sequence of stopping times along which the image of the whole circle by $K$ is reduced to $P$. We also prove that the supports of these flows contains a finite number of points, and that except for some particular cases this number of points can be arbitrarily large."
   },
   "state_tokens": 189,
   "tier": "common",
   "primary": "archive",
   "questions": [
    {
     "id": "archive",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Which arXiv archive (top-level subject group) is this paper's primary category in?",
     "options": [
      [
       "astro_ph",
       "Astrophysics"
      ],
      [
       "cond_mat",
       "Condensed matter physics"
      ],
      [
       "cs",
       "Computer science"
      ],
      [
       "econ",
       "Economics"
      ],
      [
       "eess",
       "Electrical engineering and systems science"
      ],
      [
       "math",
       "Mathematics"
      ],
      [
       "nlin",
       "Nonlinear sciences"
      ],
      [
       "physics",
       "Physics (general physics archive: optics, fluids, plasma, atomic, chemical, applied, ...)"
      ],
      [
       "q_bio",
       "Quantitative biology"
      ],
      [
       "q_fin",
       "Quantitative finance"
      ],
      [
       "stat",
       "Statistics"
      ]
     ],
     "graded": true,
     "expected": "math",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.617,
        "nlin": 0.2999,
        "stat": 0.0418,
        "physics": 0.0145,
        "cs": 0.0064,
        "cond_mat": 0.0055
       },
       "probability_of_expected": 0.617,
       "proper_score": 0.8806
      },
      "Jev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.99,
        "nlin": 0.01,
        "cs": 0.0,
        "eess": 0.0,
        "astro_ph": 0.0,
        "cond_mat": 0.0
       },
       "probability_of_expected": 0.99,
       "proper_score": 0.9999
      },
      "JevFish": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.5952,
        "nlin": 0.3177,
        "stat": 0.0438,
        "astro_ph": 0.009,
        "cond_mat": 0.0089,
        "physics": 0.0084
       },
       "probability_of_expected": 0.5952,
       "proper_score": 0.8665
      },
      "NanoJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.5078,
        "stat": 0.2578,
        "physics": 0.0531,
        "nlin": 0.0358,
        "econ": 0.0336,
        "q_fin": 0.0326
       },
       "probability_of_expected": 0.5078,
       "proper_score": 0.8417
      },
      "OpenJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.9963,
        "stat": 0.0016,
        "nlin": 0.0005,
        "astro_ph": 0.0005,
        "q_bio": 0.0004,
        "eess": 0.0003
       },
       "probability_of_expected": 0.9963,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.3658,
        "nlin": 0.1653,
        "stat": 0.1437,
        "q_fin": 0.0873,
        "econ": 0.061,
        "cond_mat": 0.0477
       },
       "probability_of_expected": 0.3658,
       "proper_score": 0.7661
      }
     }
    },
    {
     "id": "subcategory",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Within the paper's true archive, which subcategory is its primary category? The options are that archive's subcategories.",
     "options": [
      [
       "math_ac",
       "math.ac"
      ],
      [
       "math_ag",
       "math.ag"
      ],
      [
       "math_ap",
       "math.ap"
      ],
      [
       "math_at",
       "math.at"
      ],
      [
       "math_ca",
       "math.ca"
      ],
      [
       "math_co",
       "math.co"
      ],
      [
       "math_ct",
       "math.ct"
      ],
      [
       "math_cv",
       "math.cv"
      ],
      [
       "math_dg",
       "math.dg"
      ],
      [
       "math_ds",
       "math.ds"
      ],
      [
       "math_fa",
       "math.fa"
      ],
      [
       "math_gm",
       "math.gm"
      ],
      [
       "math_gn",
       "math.gn"
      ],
      [
       "math_gr",
       "math.gr"
      ],
      [
       "math_gt",
       "math.gt"
      ],
      [
       "math_ho",
       "math.ho"
      ],
      [
       "math_kt",
       "math.kt"
      ],
      [
       "math_lo",
       "math.lo"
      ],
      [
       "math_mg",
       "math.mg"
      ],
      [
       "math_na",
       "math.na"
      ],
      [
       "math_nt",
       "math.nt"
      ],
      [
       "math_oa",
       "math.oa"
      ],
      [
       "math_oc",
       "math.oc"
      ],
      [
       "math_pr",
       "math.pr"
      ],
      [
       "math_qa",
       "math.qa"
      ],
      [
       "math_ra",
       "math.ra"
      ],
      [
       "math_rt",
       "math.rt"
      ],
      [
       "math_sg",
       "math.sg"
      ],
      [
       "math_sp",
       "math.sp"
      ],
      [
       "math_st",
       "math.st"
      ]
     ],
     "graded": true,
     "expected": "math_pr",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.3037,
        "math_ag": 0.1407,
        "math_ap": 0.0621,
        "math_ca": 0.0621,
        "math_dg": 0.0564,
        "math_at": 0.0384,
        "math_pr": 0.0249
       },
       "probability_of_expected": 0.0249,
       "proper_score": 0.4593
      },
      "Jev": {
       "answered": true,
       "pick": "math_st",
       "correct": false,
       "probabilities": {
        "math_st": 0.36,
        "math_ds": 0.33,
        "math_sp": 0.22,
        "math_sg": 0.03,
        "math_pr": 0.02,
        "math_ra": 0.02
       },
       "probability_of_expected": 0.02,
       "proper_score": 0.3756
      },
      "JevFish": {
       "answered": true,
       "pick": "math_st",
       "correct": false,
       "probabilities": {
        "math_st": 0.2258,
        "math_ac": 0.1444,
        "math_ag": 0.058,
        "math_sg": 0.0575,
        "math_sp": 0.0472,
        "math_ap": 0.0469,
        "math_pr": 0.0275
       },
       "probability_of_expected": 0.0275,
       "proper_score": 0.4821
      },
      "NanoJev": {
       "answered": true,
       "pick": "math_qa",
       "correct": false,
       "probabilities": {
        "math_qa": 0.0482,
        "math_gn": 0.0453,
        "math_dg": 0.0429,
        "math_gm": 0.0424,
        "math_mg": 0.0424,
        "math_nt": 0.0421,
        "math_pr": 0.0306
       },
       "probability_of_expected": 0.0306,
       "proper_score": 0.5132
      },
      "OpenJev": {
       "answered": true,
       "pick": "math_st",
       "correct": false,
       "probabilities": {
        "math_st": 0.4423,
        "math_gm": 0.3131,
        "math_at": 0.0692,
        "math_ds": 0.0431,
        "math_ca": 0.015,
        "math_ct": 0.0123,
        "math_pr": 0.0021
       },
       "probability_of_expected": 0.0021,
       "proper_score": 0.3514
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math_st",
       "correct": false,
       "probabilities": {
        "math_st": 0.1065,
        "math_pr": 0.0784,
        "math_ct": 0.063,
        "math_oc": 0.0601,
        "math_ag": 0.0593,
        "math_oa": 0.0509
       },
       "probability_of_expected": 0.0784,
       "proper_score": 0.5541
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 5650.6,
    "Jev": 300.0,
    "JevFish": 2345.0,
    "NanoJev": 1924.0,
    "OpenJev": 418.4,
    "System One Scorer": 10550.0
   }
  },
  {
   "id": "data-and-operations:arxiv-baf88e3ff5a7",
   "state": {
    "title": "Asymptotics of the $s$-perimeter as $s\\searrow 0$",
    "abstract": "We deal with the asymptotic behavior of the $s$-perimeter of a set $E$ inside a domain $\\Omega$ as $s\\searrow0$. We prove necessary and sufficient conditions for the existence of such limit, by also providing an explicit formulation in terms of the Lebesgue measure of $E$ and $\\Omega$. Moreover, we construct examples of sets for which the limit does not exist."
   },
   "state_tokens": 115,
   "tier": "common",
   "primary": "archive",
   "questions": [
    {
     "id": "archive",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Which arXiv archive (top-level subject group) is this paper's primary category in?",
     "options": [
      [
       "astro_ph",
       "Astrophysics"
      ],
      [
       "cond_mat",
       "Condensed matter physics"
      ],
      [
       "cs",
       "Computer science"
      ],
      [
       "econ",
       "Economics"
      ],
      [
       "eess",
       "Electrical engineering and systems science"
      ],
      [
       "math",
       "Mathematics"
      ],
      [
       "nlin",
       "Nonlinear sciences"
      ],
      [
       "physics",
       "Physics (general physics archive: optics, fluids, plasma, atomic, chemical, applied, ...)"
      ],
      [
       "q_bio",
       "Quantitative biology"
      ],
      [
       "q_fin",
       "Quantitative finance"
      ],
      [
       "stat",
       "Statistics"
      ]
     ],
     "graded": true,
     "expected": "math",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.8812,
        "nlin": 0.0877,
        "stat": 0.0128,
        "physics": 0.0062,
        "cs": 0.0033,
        "eess": 0.002
       },
       "probability_of_expected": 0.8812,
       "proper_score": 0.989
      },
      "Jev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 1.0,
        "physics": 0.0,
        "econ": 0.0,
        "cs": 0.0,
        "q_fin": 0.0,
        "astro_ph": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.8327,
        "nlin": 0.1133,
        "stat": 0.0193,
        "astro_ph": 0.0096,
        "physics": 0.006,
        "eess": 0.005
       },
       "probability_of_expected": 0.8327,
       "proper_score": 0.9793
      },
      "NanoJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.5286,
        "stat": 0.2341,
        "physics": 0.0775,
        "cs": 0.0433,
        "econ": 0.0376,
        "nlin": 0.0239
       },
       "probability_of_expected": 0.5286,
       "proper_score": 0.8562
      },
      "OpenJev": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.9994,
        "nlin": 0.0002,
        "stat": 0.0001,
        "cond_mat": 0.0001,
        "eess": 0.0,
        "astro_ph": 0.0
       },
       "probability_of_expected": 0.9994,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math",
       "correct": true,
       "probabilities": {
        "math": 0.552,
        "cond_mat": 0.0737,
        "econ": 0.0727,
        "nlin": 0.0536,
        "physics": 0.0469,
        "stat": 0.0469
       },
       "probability_of_expected": 0.552,
       "proper_score": 0.888
      }
     }
    },
    {
     "id": "subcategory",
     "type": "choice",
     "instructions": "Read state.title and state.abstract. Within the paper's true archive, which subcategory is its primary category? The options are that archive's subcategories.",
     "options": [
      [
       "math_ac",
       "math.ac"
      ],
      [
       "math_ag",
       "math.ag"
      ],
      [
       "math_ap",
       "math.ap"
      ],
      [
       "math_at",
       "math.at"
      ],
      [
       "math_ca",
       "math.ca"
      ],
      [
       "math_co",
       "math.co"
      ],
      [
       "math_ct",
       "math.ct"
      ],
      [
       "math_cv",
       "math.cv"
      ],
      [
       "math_dg",
       "math.dg"
      ],
      [
       "math_ds",
       "math.ds"
      ],
      [
       "math_fa",
       "math.fa"
      ],
      [
       "math_gm",
       "math.gm"
      ],
      [
       "math_gn",
       "math.gn"
      ],
      [
       "math_gr",
       "math.gr"
      ],
      [
       "math_gt",
       "math.gt"
      ],
      [
       "math_ho",
       "math.ho"
      ],
      [
       "math_kt",
       "math.kt"
      ],
      [
       "math_lo",
       "math.lo"
      ],
      [
       "math_mg",
       "math.mg"
      ],
      [
       "math_na",
       "math.na"
      ],
      [
       "math_nt",
       "math.nt"
      ],
      [
       "math_oa",
       "math.oa"
      ],
      [
       "math_oc",
       "math.oc"
      ],
      [
       "math_pr",
       "math.pr"
      ],
      [
       "math_qa",
       "math.qa"
      ],
      [
       "math_ra",
       "math.ra"
      ],
      [
       "math_rt",
       "math.rt"
      ],
      [
       "math_sg",
       "math.sg"
      ],
      [
       "math_sp",
       "math.sp"
      ],
      [
       "math_st",
       "math.st"
      ]
     ],
     "graded": true,
     "expected": "math_ap",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.2337,
        "math_ag": 0.1137,
        "math_dg": 0.0812,
        "math_ca": 0.0703,
        "math_ap": 0.0638,
        "math_at": 0.0456
       },
       "probability_of_expected": 0.0638,
       "proper_score": 0.5167
      },
      "Jev": {
       "answered": true,
       "pick": "math_mg",
       "correct": false,
       "probabilities": {
        "math_mg": 0.46,
        "math_ca": 0.26,
        "math_cv": 0.07,
        "math_oc": 0.05,
        "math_ac": 0.04,
        "math_dg": 0.03,
        "math_ap": 0.02
       },
       "probability_of_expected": 0.02,
       "proper_score": 0.3745
      },
      "JevFish": {
       "answered": true,
       "pick": "math_ac",
       "correct": false,
       "probabilities": {
        "math_ac": 0.1851,
        "math_st": 0.1751,
        "math_sg": 0.0653,
        "math_ag": 0.059,
        "math_sp": 0.0539,
        "math_ap": 0.0495
       },
       "probability_of_expected": 0.0495,
       "proper_score": 0.5067
      },
      "NanoJev": {
       "answered": true,
       "pick": "math_qa",
       "correct": false,
       "probabilities": {
        "math_qa": 0.0592,
        "math_gn": 0.0551,
        "math_gm": 0.0456,
        "math_dg": 0.0453,
        "math_oa": 0.0449,
        "math_mg": 0.0444,
        "math_ap": 0.0313
       },
       "probability_of_expected": 0.0313,
       "proper_score": 0.513
      },
      "OpenJev": {
       "answered": true,
       "pick": "math_ct",
       "correct": false,
       "probabilities": {
        "math_ct": 0.3801,
        "math_gm": 0.1346,
        "math_st": 0.1262,
        "math_fa": 0.0756,
        "math_sp": 0.0464,
        "math_ra": 0.0367,
        "math_ap": 0.0279
       },
       "probability_of_expected": 0.0279,
       "proper_score": 0.4323
      },
      "System One Scorer": {
       "answered": true,
       "pick": "math_ag",
       "correct": false,
       "probabilities": {
        "math_ag": 0.1728,
        "math_mg": 0.0654,
        "math_st": 0.0642,
        "math_gr": 0.0625,
        "math_ac": 0.0526,
        "math_co": 0.0451,
        "math_ap": 0.0301
       },
       "probability_of_expected": 0.0301,
       "proper_score": 0.4992
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4946.0,
    "Jev": 157.2,
    "JevFish": 2392.2,
    "NanoJev": 1895.9,
    "OpenJev": 477.5,
    "System One Scorer": 10411.6
   }
  }
 ]
}