{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "dbpedia-ontology-hierarchy",
 "family": "data-and-operations",
 "use_case": "hierarchical-classification",
 "vertical": "media-and-reference",
 "dataset": "DeveloperOats/DBPedia_Classes",
 "licence": "CC0-1.0",
 "provenance": "outcome",
 "state_kind": "text",
 "primitive": "choice",
 "items": 200,
 "scores": {
  "decider-2b": {
   "decisionScore": 89.04,
   "accuracy": 0.89,
   "answered": 200,
   "items": 200
  },
  "Jev": {
   "decisionScore": 92.86,
   "accuracy": 0.94,
   "answered": 200,
   "items": 200
  },
  "JevFish": {
   "decisionScore": 89.37,
   "accuracy": 0.885,
   "answered": 200,
   "items": 200
  },
  "NanoJev": {
   "decisionScore": 60.64,
   "accuracy": 0.155,
   "answered": 200,
   "items": 200
  },
  "OpenJev": {
   "decisionScore": 89.31,
   "accuracy": 0.925,
   "answered": 200,
   "items": 200
  },
  "System One Scorer": {
   "decisionScore": 83.82,
   "accuracy": 0.87,
   "answered": 200,
   "items": 200
  }
 },
 "samples": [
  {
   "id": "data-and-operations:dbpedia-3f10e15ec7dd",
   "state": {
    "article_text": "The 2010–11 football season is Bristol City's 113th season as a professional football club and fourth consecutive season in the second division. They are competing in the Football League Championship having finished in tenth position the previous season. It was Steve Coppell's first season in charge of the club, though he resigned as manager on 12 August 2010 after just one league game in charge. Assistant Keith Millen became the club's new manager on a three-year-deal and it is his first job as a manager."
   },
   "state_tokens": 120,
   "tier": "common",
   "primary": "l1",
   "questions": [
    {
     "id": "l1",
     "type": "choice",
     "instructions": "Read state.article_text (the opening of a Wikipedia article). Which top-level DBpedia ontology class does the article's subject belong to?",
     "options": [
      [
       "agent",
       "A person, group, organisation, company, team, character or other acting entity"
      ],
      [
       "place",
       "A geographic or built location: settlement, building, natural feature, route, station, celestial body"
      ],
      [
       "species",
       "A biological organism: animal, plant, fungus or other eukaryote (incl. named breeds)"
      ],
      [
       "work",
       "A created work: album, song, book, periodical, software, comic, cartoon"
      ],
      [
       "event",
       "A dated happening: tournament, race, sports event, societal event, natural event, Olympics"
      ],
      [
       "sports_season",
       "One season of a sports team or league"
      ],
      [
       "unit_of_work",
       "A legal case or similar unit of proceedings"
      ],
      [
       "device",
       "A manufactured device such as an engine"
      ],
      [
       "topical_concept",
       "An abstract topic such as a genre"
      ]
     ],
     "graded": true,
     "expected": "sports_season",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "agent",
       "correct": false,
       "probabilities": {
        "agent": 0.7573,
        "sports_season": 0.1971,
        "event": 0.0226,
        "topical_concept": 0.0059,
        "species": 0.0042,
        "work": 0.0042
       },
       "probability_of_expected": 0.1971,
       "proper_score": 0.3907
      },
      "Jev": {
       "answered": true,
       "pick": "sports_season",
       "correct": true,
       "probabilities": {
        "sports_season": 1.0,
        "species": 0.0,
        "place": 0.0,
        "event": 0.0,
        "device": 0.0,
        "work": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "sports_season",
       "correct": true,
       "probabilities": {
        "sports_season": 0.9954,
        "agent": 0.0032,
        "event": 0.0004,
        "topical_concept": 0.0003,
        "place": 0.0002,
        "species": 0.0002
       },
       "probability_of_expected": 0.9954,
       "proper_score": 1.0
      },
      "NanoJev": {
       "answered": true,
       "pick": "sports_season",
       "correct": true,
       "probabilities": {
        "sports_season": 0.3114,
        "unit_of_work": 0.2663,
        "agent": 0.2546,
        "topical_concept": 0.0901,
        "event": 0.0338,
        "place": 0.0287
       },
       "probability_of_expected": 0.3114,
       "proper_score": 0.6899
      },
      "OpenJev": {
       "answered": true,
       "pick": "event",
       "correct": false,
       "probabilities": {
        "event": 0.6585,
        "sports_season": 0.3327,
        "agent": 0.008,
        "place": 0.0005,
        "work": 0.0001,
        "topical_concept": 0.0001
       },
       "probability_of_expected": 0.3327,
       "proper_score": 0.5605
      },
      "System One Scorer": {
       "answered": true,
       "pick": "agent",
       "correct": false,
       "probabilities": {
        "agent": 0.4464,
        "event": 0.1827,
        "sports_season": 0.1547,
        "place": 0.0697,
        "species": 0.052,
        "topical_concept": 0.0344
       },
       "probability_of_expected": 0.1547,
       "proper_score": 0.5213
      }
     }
    },
    {
     "id": "l2",
     "type": "choice",
     "instructions": "Read state.article_text. Within the top-level class you chose, which second-level DBpedia class fits best? The options are the second-level classes of the article's true top-level class.",
     "options": [
      [
       "football_league_season",
       "one season of a football league"
      ],
      [
       "sports_team_season",
       "one season of a sports team"
      ]
     ],
     "graded": true,
     "expected": "sports_team_season",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "sports_team_season",
       "correct": true,
       "probabilities": {
        "sports_team_season": 0.8368,
        "football_league_season": 0.1632
       },
       "probability_of_expected": 0.8368,
       "proper_score": 0.9734
      },
      "Jev": {
       "answered": true,
       "pick": "sports_team_season",
       "correct": true,
       "probabilities": {
        "sports_team_season": 0.91,
        "football_league_season": 0.09
       },
       "probability_of_expected": 0.91,
       "proper_score": 0.9919
      },
      "JevFish": {
       "answered": true,
       "pick": "football_league_season",
       "correct": false,
       "probabilities": {
        "football_league_season": 0.5644,
        "sports_team_season": 0.4356
       },
       "probability_of_expected": 0.4356,
       "proper_score": 0.6814
      },
      "NanoJev": {
       "answered": true,
       "pick": "football_league_season",
       "correct": false,
       "probabilities": {
        "football_league_season": 0.7264,
        "sports_team_season": 0.2736
       },
       "probability_of_expected": 0.2736,
       "proper_score": 0.4723
      },
      "OpenJev": {
       "answered": true,
       "pick": "sports_team_season",
       "correct": true,
       "probabilities": {
        "sports_team_season": 0.8574,
        "football_league_season": 0.1426
       },
       "probability_of_expected": 0.8574,
       "proper_score": 0.9797
      },
      "System One Scorer": {
       "answered": true,
       "pick": "sports_team_season",
       "correct": true,
       "probabilities": {
        "sports_team_season": 0.6237,
        "football_league_season": 0.3763
       },
       "probability_of_expected": 0.6237,
       "proper_score": 0.8584
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 3152.9,
    "Jev": 244.1,
    "JevFish": 2259.2,
    "NanoJev": 1182.1,
    "OpenJev": 340.8,
    "System One Scorer": 6001.7
   }
  },
  {
   "id": "data-and-operations:dbpedia-49266c80b7d8",
   "state": {
    "article_text": "Carl Folta is Executive Vice President, Corporate Communications for Viacom. He has served at this post since November 2006. Before that, he served as Executive Vice President, Office of the Chairman, from January 1, 2006, where he served as Sumner Redstone's senior adviser and spokesman. Previously, he was Executive Vice President, Corporate Relations of the former Viacom Inc., since November 2004. Prior to that, he served as Senior Vice President of Corporate Relations of Viacom from November 1994 to November 2004, and Vice President of Corporate Relations of Viacom from April 1994 to November 1994. Folta held various communications positions at Paramount Communications from 1984 (when the company was known as Gulf+Western, retaining this name until 1989) until joining Viacom through its purchase of Paramount in April 1994."
   },
   "state_tokens": 206,
   "tier": "common",
   "primary": "l1",
   "questions": [
    {
     "id": "l1",
     "type": "choice",
     "instructions": "Read state.article_text (the opening of a Wikipedia article). Which top-level DBpedia ontology class does the article's subject belong to?",
     "options": [
      [
       "agent",
       "A person, group, organisation, company, team, character or other acting entity"
      ],
      [
       "place",
       "A geographic or built location: settlement, building, natural feature, route, station, celestial body"
      ],
      [
       "species",
       "A biological organism: animal, plant, fungus or other eukaryote (incl. named breeds)"
      ],
      [
       "work",
       "A created work: album, song, book, periodical, software, comic, cartoon"
      ],
      [
       "event",
       "A dated happening: tournament, race, sports event, societal event, natural event, Olympics"
      ],
      [
       "sports_season",
       "One season of a sports team or league"
      ],
      [
       "unit_of_work",
       "A legal case or similar unit of proceedings"
      ],
      [
       "device",
       "A manufactured device such as an engine"
      ],
      [
       "topical_concept",
       "An abstract topic such as a genre"
      ]
     ],
     "graded": true,
     "expected": "agent",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.9935,
        "work": 0.0014,
        "topical_concept": 0.0012,
        "place": 0.0008,
        "species": 0.0008,
        "event": 0.0007
       },
       "probability_of_expected": 0.9935,
       "proper_score": 1.0
      },
      "Jev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 1.0,
        "sports_season": 0.0,
        "device": 0.0,
        "place": 0.0,
        "event": 0.0,
        "topical_concept": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.9889,
        "work": 0.0033,
        "device": 0.0016,
        "place": 0.0015,
        "event": 0.0013,
        "topical_concept": 0.0012
       },
       "probability_of_expected": 0.9889,
       "proper_score": 0.9999
      },
      "NanoJev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.395,
        "unit_of_work": 0.2492,
        "topical_concept": 0.1406,
        "place": 0.0779,
        "species": 0.0538,
        "event": 0.0396
       },
       "probability_of_expected": 0.395,
       "proper_score": 0.7704
      },
      "OpenJev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 1.0,
        "place": 0.0,
        "event": 0.0,
        "work": 0.0,
        "topical_concept": 0.0,
        "species": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.7841,
        "place": 0.0441,
        "event": 0.0429,
        "work": 0.0386,
        "species": 0.0288,
        "topical_concept": 0.0262
       },
       "probability_of_expected": 0.7841,
       "proper_score": 0.9731
      }
     }
    },
    {
     "id": "l2",
     "type": "choice",
     "instructions": "Read state.article_text. Within the top-level class you chose, which second-level DBpedia class fits best? The options are the second-level classes of the article's true top-level class.",
     "options": [
      [
       "actor",
       "film, television or stage actor"
      ],
      [
       "artist",
       "visual or performing artist not covered by a more specific class"
      ],
      [
       "athlete",
       "sportsperson (generic, when no sport-specific class applies)"
      ],
      [
       "boxer",
       "boxer"
      ],
      [
       "british_royalty",
       "member of the British royal family"
      ],
      [
       "broadcaster",
       "radio or television station or network"
      ],
      [
       "cleric",
       "religious office holder"
      ],
      [
       "coach",
       "sports coach"
      ],
      [
       "company",
       "commercial company"
      ],
      [
       "educational_institution",
       "school, college or university"
      ],
      [
       "fictional_character",
       "fictional character (non-comics)"
      ],
      [
       "gridiron_football_player",
       "American or Canadian football player"
      ],
      [
       "group",
       "band or other group of people"
      ],
      [
       "motorcycle_rider",
       "motorcycle racer"
      ],
      [
       "organisation",
       "organisation not covered by a more specific class"
      ],
      [
       "organisation_member",
       "organisation member"
      ],
      [
       "person",
       "person not covered by a more specific class"
      ],
      [
       "politician",
       "politician or office holder"
      ],
      [
       "racing_driver",
       "motor racing driver"
      ],
      [
       "scientist",
       "scientist or academic"
      ],
      [
       "sports_league",
       "sports league"
      ],
      [
       "sports_manager",
       "sports manager"
      ],
      [
       "sports_team",
       "sports team or club"
      ],
      [
       "winter_sport_player",
       "winter-sport athlete (ice hockey, skiing, ...)"
      ],
      [
       "wrestler",
       "wrestler"
      ],
      [
       "writer",
       "author, poet or journalist"
      ]
     ],
     "graded": true,
     "expected": "person",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "person",
       "correct": true,
       "probabilities": {
        "person": 0.885,
        "organisation": 0.0337,
        "organisation_member": 0.0265,
        "company": 0.0117,
        "politician": 0.0072,
        "wrestler": 0.0057
       },
       "probability_of_expected": 0.885,
       "proper_score": 0.9923
      },
      "Jev": {
       "answered": true,
       "pick": "person",
       "correct": true,
       "probabilities": {
        "person": 0.72,
        "organisation_member": 0.23,
        "politician": 0.02,
        "company": 0.02,
        "broadcaster": 0.01,
        "boxer": 0.0
       },
       "probability_of_expected": 0.72,
       "proper_score": 0.9339
      },
      "JevFish": {
       "answered": true,
       "pick": "person",
       "correct": true,
       "probabilities": {
        "person": 0.4906,
        "organisation": 0.1178,
        "politician": 0.1055,
        "organisation_member": 0.1011,
        "company": 0.047,
        "educational_institution": 0.0127
       },
       "probability_of_expected": 0.4906,
       "proper_score": 0.851
      },
      "NanoJev": {
       "answered": true,
       "pick": "organisation_member",
       "correct": false,
       "probabilities": {
        "organisation_member": 0.1565,
        "person": 0.1277,
        "organisation": 0.1101,
        "artist": 0.1059,
        "company": 0.0983,
        "group": 0.065
       },
       "probability_of_expected": 0.1277,
       "proper_score": 0.5842
      },
      "OpenJev": {
       "answered": true,
       "pick": "person",
       "correct": true,
       "probabilities": {
        "person": 0.9984,
        "organisation_member": 0.0006,
        "politician": 0.0003,
        "organisation": 0.0002,
        "artist": 0.0001,
        "actor": 0.0001
       },
       "probability_of_expected": 0.9984,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "organisation_member",
       "correct": false,
       "probabilities": {
        "organisation_member": 0.35,
        "broadcaster": 0.2042,
        "company": 0.1273,
        "organisation": 0.0849,
        "person": 0.0738,
        "politician": 0.0337
       },
       "probability_of_expected": 0.0738,
       "proper_score": 0.4762
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4758.1,
    "Jev": 167.9,
    "JevFish": 2363.7,
    "NanoJev": 1266.4,
    "OpenJev": 377.9,
    "System One Scorer": 6291.7
   }
  },
  {
   "id": "data-and-operations:dbpedia-588ab13144e8",
   "state": {
    "article_text": "The Apeldoorn–Deventer railway is an important railway line in the Netherlands, running from Apeldoorn to Deventer. The line was opened between 1887 and 1888. It is used by long distance trains from Amsterdam, Rotterdam and The Hague towards Enschede and Berlin, and by local services between Apeldoorn and Enschede."
   },
   "state_tokens": 84,
   "tier": "common",
   "primary": "l1",
   "questions": [
    {
     "id": "l1",
     "type": "choice",
     "instructions": "Read state.article_text (the opening of a Wikipedia article). Which top-level DBpedia ontology class does the article's subject belong to?",
     "options": [
      [
       "agent",
       "A person, group, organisation, company, team, character or other acting entity"
      ],
      [
       "place",
       "A geographic or built location: settlement, building, natural feature, route, station, celestial body"
      ],
      [
       "species",
       "A biological organism: animal, plant, fungus or other eukaryote (incl. named breeds)"
      ],
      [
       "work",
       "A created work: album, song, book, periodical, software, comic, cartoon"
      ],
      [
       "event",
       "A dated happening: tournament, race, sports event, societal event, natural event, Olympics"
      ],
      [
       "sports_season",
       "One season of a sports team or league"
      ],
      [
       "unit_of_work",
       "A legal case or similar unit of proceedings"
      ],
      [
       "device",
       "A manufactured device such as an engine"
      ],
      [
       "topical_concept",
       "An abstract topic such as a genre"
      ]
     ],
     "graded": true,
     "expected": "place",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "place",
       "correct": true,
       "probabilities": {
        "place": 0.8469,
        "topical_concept": 0.0473,
        "agent": 0.0338,
        "device": 0.0338,
        "event": 0.0088,
        "sports_season": 0.0088
       },
       "probability_of_expected": 0.8469,
       "proper_score": 0.9859
      },
      "Jev": {
       "answered": true,
       "pick": "place",
       "correct": true,
       "probabilities": {
        "place": 1.0,
        "unit_of_work": 0.0,
        "device": 0.0,
        "event": 0.0,
        "work": 0.0,
        "topical_concept": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "place",
       "correct": true,
       "probabilities": {
        "place": 0.9886,
        "topical_concept": 0.0028,
        "work": 0.0017,
        "species": 0.0017,
        "event": 0.0017,
        "sports_season": 0.0014
       },
       "probability_of_expected": 0.9886,
       "proper_score": 0.9999
      },
      "NanoJev": {
       "answered": true,
       "pick": "unit_of_work",
       "correct": false,
       "probabilities": {
        "unit_of_work": 0.3802,
        "agent": 0.2637,
        "topical_concept": 0.161,
        "place": 0.0969,
        "species": 0.0442,
        "event": 0.0312
       },
       "probability_of_expected": 0.0969,
       "proper_score": 0.4706
      },
      "OpenJev": {
       "answered": true,
       "pick": "place",
       "correct": true,
       "probabilities": {
        "place": 0.9986,
        "agent": 0.0007,
        "device": 0.0003,
        "sports_season": 0.0002,
        "topical_concept": 0.0001,
        "event": 0.0
       },
       "probability_of_expected": 0.9986,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "place",
       "correct": true,
       "probabilities": {
        "place": 0.5374,
        "agent": 0.1072,
        "event": 0.0821,
        "work": 0.0729,
        "topical_concept": 0.0559,
        "species": 0.0537
       },
       "probability_of_expected": 0.5374,
       "proper_score": 0.8768
      }
     }
    },
    {
     "id": "l2",
     "type": "choice",
     "instructions": "Read state.article_text. Within the top-level class you chose, which second-level DBpedia class fits best? The options are the second-level classes of the article's true top-level class.",
     "options": [
      [
       "amusement_park_attraction",
       "ride or attraction in an amusement park"
      ],
      [
       "body_of_water",
       "lake, bay, sea or other still water body"
      ],
      [
       "building",
       "building (house, church, museum, ...)"
      ],
      [
       "celestial_body",
       "star, planet, asteroid or galaxy"
      ],
      [
       "clerical_administrative_region",
       "diocese or other church administrative region"
      ],
      [
       "infrastructure",
       "dam, airport, power station, bridge or similar"
      ],
      [
       "natural_place",
       "mountain, valley, cave, glacier or other landform"
      ],
      [
       "route_of_transportation",
       "road, railway line or public transit route"
      ],
      [
       "satellite",
       "artificial satellite"
      ],
      [
       "settlement",
       "city, town or village"
      ],
      [
       "sport_facility",
       "stadium, arena or golf course"
      ],
      [
       "station",
       "railway or metro station"
      ],
      [
       "stream",
       "river or stream"
      ],
      [
       "tower",
       "tower or lighthouse"
      ],
      [
       "venue",
       "theatre, concert hall or other venue"
      ]
     ],
     "graded": true,
     "expected": "route_of_transportation",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "route_of_transportation",
       "correct": true,
       "probabilities": {
        "route_of_transportation": 0.9773,
        "infrastructure": 0.0102,
        "station": 0.0052,
        "settlement": 0.0029,
        "venue": 0.0009,
        "building": 0.0006
       },
       "probability_of_expected": 0.9773,
       "proper_score": 0.9997
      },
      "Jev": {
       "answered": true,
       "pick": "route_of_transportation",
       "correct": true,
       "probabilities": {
        "route_of_transportation": 0.98,
        "infrastructure": 0.02,
        "station": 0.0,
        "satellite": 0.0,
        "celestial_body": 0.0,
        "building": 0.0
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9996
      },
      "JevFish": {
       "answered": true,
       "pick": "route_of_transportation",
       "correct": true,
       "probabilities": {
        "route_of_transportation": 0.7503,
        "station": 0.181,
        "infrastructure": 0.0088,
        "amusement_park_attraction": 0.0087,
        "venue": 0.0072,
        "building": 0.0058
       },
       "probability_of_expected": 0.7503,
       "proper_score": 0.9523
      },
      "NanoJev": {
       "answered": true,
       "pick": "station",
       "correct": false,
       "probabilities": {
        "station": 0.4883,
        "route_of_transportation": 0.2249,
        "settlement": 0.0494,
        "body_of_water": 0.0436,
        "natural_place": 0.0423,
        "celestial_body": 0.0361
       },
       "probability_of_expected": 0.2249,
       "proper_score": 0.5755
      },
      "OpenJev": {
       "answered": true,
       "pick": "route_of_transportation",
       "correct": true,
       "probabilities": {
        "route_of_transportation": 0.9969,
        "infrastructure": 0.0026,
        "station": 0.0001,
        "body_of_water": 0.0001,
        "natural_place": 0.0001,
        "sport_facility": 0.0001
       },
       "probability_of_expected": 0.9969,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "route_of_transportation",
       "correct": true,
       "probabilities": {
        "route_of_transportation": 0.8523,
        "settlement": 0.0365,
        "infrastructure": 0.0185,
        "station": 0.0158,
        "clerical_administrative_region": 0.0116,
        "venue": 0.0107
       },
       "probability_of_expected": 0.8523,
       "proper_score": 0.9878
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4451.7,
    "Jev": 288.5,
    "JevFish": 2402.4,
    "NanoJev": 1178.8,
    "OpenJev": 330.8,
    "System One Scorer": 5617.7
   }
  },
  {
   "id": "data-and-operations:dbpedia-82cdaa7294bf",
   "state": {
    "article_text": "Electroclash, also known as retro electro, nouveau disco, the new new wave, and tech-pop is a genre of music that fuses 1980s electro, new wave and synthpop with 1990s techno, retro-style electropop and electronic dance music. It emerged in the later 1990s, pioneered by and associated with acts such as for example I-F, Miss Kittin & The Hacker or Fischerspooner."
   },
   "state_tokens": 105,
   "tier": "common",
   "primary": "l1",
   "questions": [
    {
     "id": "l1",
     "type": "choice",
     "instructions": "Read state.article_text (the opening of a Wikipedia article). Which top-level DBpedia ontology class does the article's subject belong to?",
     "options": [
      [
       "agent",
       "A person, group, organisation, company, team, character or other acting entity"
      ],
      [
       "place",
       "A geographic or built location: settlement, building, natural feature, route, station, celestial body"
      ],
      [
       "species",
       "A biological organism: animal, plant, fungus or other eukaryote (incl. named breeds)"
      ],
      [
       "work",
       "A created work: album, song, book, periodical, software, comic, cartoon"
      ],
      [
       "event",
       "A dated happening: tournament, race, sports event, societal event, natural event, Olympics"
      ],
      [
       "sports_season",
       "One season of a sports team or league"
      ],
      [
       "unit_of_work",
       "A legal case or similar unit of proceedings"
      ],
      [
       "device",
       "A manufactured device such as an engine"
      ],
      [
       "topical_concept",
       "An abstract topic such as a genre"
      ]
     ],
     "graded": true,
     "expected": "topical_concept",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "topical_concept",
       "correct": true,
       "probabilities": {
        "topical_concept": 0.9972,
        "work": 0.0013,
        "agent": 0.0004,
        "event": 0.0003,
        "place": 0.0002,
        "species": 0.0002
       },
       "probability_of_expected": 0.9972,
       "proper_score": 1.0
      },
      "Jev": {
       "answered": true,
       "pick": "topical_concept",
       "correct": true,
       "probabilities": {
        "topical_concept": 1.0,
        "species": 0.0,
        "work": 0.0,
        "place": 0.0,
        "device": 0.0,
        "unit_of_work": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "topical_concept",
       "correct": true,
       "probabilities": {
        "topical_concept": 0.9991,
        "work": 0.0003,
        "unit_of_work": 0.0002,
        "agent": 0.0001,
        "event": 0.0001,
        "place": 0.0001
       },
       "probability_of_expected": 0.9991,
       "proper_score": 1.0
      },
      "NanoJev": {
       "answered": true,
       "pick": "topical_concept",
       "correct": true,
       "probabilities": {
        "topical_concept": 0.3332,
        "unit_of_work": 0.31,
        "agent": 0.2037,
        "place": 0.0511,
        "event": 0.0314,
        "device": 0.0314
       },
       "probability_of_expected": 0.3332,
       "proper_score": 0.7062
      },
      "OpenJev": {
       "answered": true,
       "pick": "topical_concept",
       "correct": true,
       "probabilities": {
        "topical_concept": 0.9977,
        "work": 0.0017,
        "agent": 0.0003,
        "event": 0.0003,
        "place": 0.0,
        "species": 0.0
       },
       "probability_of_expected": 0.9977,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "topical_concept",
       "correct": true,
       "probabilities": {
        "topical_concept": 0.5094,
        "agent": 0.1269,
        "work": 0.1068,
        "event": 0.0935,
        "place": 0.0636,
        "species": 0.0369
       },
       "probability_of_expected": 0.5094,
       "proper_score": 0.8582
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 3550.8,
    "Jev": 261.8,
    "JevFish": 2199.2,
    "NanoJev": 1527.2,
    "OpenJev": 369.3,
    "System One Scorer": 7894.8
   }
  },
  {
   "id": "data-and-operations:dbpedia-ad98025f16c2",
   "state": {
    "article_text": "Franco Ballerini (11 December 1964 – 7 February 2010) was an Italian road racing cyclist. Born in Florence, his greatest exploits as a rider came with his two victories in the cycling classic Paris–Roubaix, riding for the Mapei cycling team. In 1993 he was beaten on the line by Gilbert Duclos-Lassalle in an exciting Paris–Roubaix finale. Also on his palmarès or list of accomplishments there are other one-day races such as Omloop Het Volk and Paris–Brussels. Ballerini then became manager of the Italian national cycling squad, winning the 2002 World Championships with Mario Cipollini and the 2004 Summer Olympics in Athens with ex-teammate Paolo Bettini. In 2006, 2007 and 2008 he won the World Championships with Paolo Bettini and Alessandro Ballan."
   },
   "state_tokens": 210,
   "tier": "common",
   "primary": "l1",
   "questions": [
    {
     "id": "l1",
     "type": "choice",
     "instructions": "Read state.article_text (the opening of a Wikipedia article). Which top-level DBpedia ontology class does the article's subject belong to?",
     "options": [
      [
       "agent",
       "A person, group, organisation, company, team, character or other acting entity"
      ],
      [
       "place",
       "A geographic or built location: settlement, building, natural feature, route, station, celestial body"
      ],
      [
       "species",
       "A biological organism: animal, plant, fungus or other eukaryote (incl. named breeds)"
      ],
      [
       "work",
       "A created work: album, song, book, periodical, software, comic, cartoon"
      ],
      [
       "event",
       "A dated happening: tournament, race, sports event, societal event, natural event, Olympics"
      ],
      [
       "sports_season",
       "One season of a sports team or league"
      ],
      [
       "unit_of_work",
       "A legal case or similar unit of proceedings"
      ],
      [
       "device",
       "A manufactured device such as an engine"
      ],
      [
       "topical_concept",
       "An abstract topic such as a genre"
      ]
     ],
     "graded": true,
     "expected": "agent",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.9769,
        "species": 0.0123,
        "event": 0.0034,
        "topical_concept": 0.002,
        "place": 0.0015,
        "work": 0.0014
       },
       "probability_of_expected": 0.9769,
       "proper_score": 0.9996
      },
      "Jev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 1.0,
        "unit_of_work": 0.0,
        "topical_concept": 0.0,
        "device": 0.0,
        "place": 0.0,
        "species": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.8411,
        "event": 0.1358,
        "sports_season": 0.0158,
        "topical_concept": 0.0016,
        "work": 0.0015,
        "device": 0.0013
       },
       "probability_of_expected": 0.8411,
       "proper_score": 0.978
      },
      "NanoJev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.2253,
        "topical_concept": 0.1973,
        "unit_of_work": 0.1946,
        "event": 0.1457,
        "sports_season": 0.1109,
        "place": 0.0759
       },
       "probability_of_expected": 0.2253,
       "proper_score": 0.6413
      },
      "OpenJev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.9997,
        "place": 0.0002,
        "work": 0.0001,
        "topical_concept": 0.0,
        "species": 0.0,
        "event": 0.0
       },
       "probability_of_expected": 0.9997,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.647,
        "event": 0.1338,
        "species": 0.0824,
        "place": 0.0591,
        "work": 0.0262,
        "topical_concept": 0.0239
       },
       "probability_of_expected": 0.647,
       "proper_score": 0.9228
      }
     }
    },
    {
     "id": "l2",
     "type": "choice",
     "instructions": "Read state.article_text. Within the top-level class you chose, which second-level DBpedia class fits best? The options are the second-level classes of the article's true top-level class.",
     "options": [
      [
       "actor",
       "film, television or stage actor"
      ],
      [
       "artist",
       "visual or performing artist not covered by a more specific class"
      ],
      [
       "athlete",
       "sportsperson (generic, when no sport-specific class applies)"
      ],
      [
       "boxer",
       "boxer"
      ],
      [
       "british_royalty",
       "member of the British royal family"
      ],
      [
       "broadcaster",
       "radio or television station or network"
      ],
      [
       "cleric",
       "religious office holder"
      ],
      [
       "coach",
       "sports coach"
      ],
      [
       "company",
       "commercial company"
      ],
      [
       "educational_institution",
       "school, college or university"
      ],
      [
       "fictional_character",
       "fictional character (non-comics)"
      ],
      [
       "gridiron_football_player",
       "American or Canadian football player"
      ],
      [
       "group",
       "band or other group of people"
      ],
      [
       "motorcycle_rider",
       "motorcycle racer"
      ],
      [
       "organisation",
       "organisation not covered by a more specific class"
      ],
      [
       "organisation_member",
       "organisation member"
      ],
      [
       "person",
       "person not covered by a more specific class"
      ],
      [
       "politician",
       "politician or office holder"
      ],
      [
       "racing_driver",
       "motor racing driver"
      ],
      [
       "scientist",
       "scientist or academic"
      ],
      [
       "sports_league",
       "sports league"
      ],
      [
       "sports_manager",
       "sports manager"
      ],
      [
       "sports_team",
       "sports team or club"
      ],
      [
       "winter_sport_player",
       "winter-sport athlete (ice hockey, skiing, ...)"
      ],
      [
       "wrestler",
       "wrestler"
      ],
      [
       "writer",
       "author, poet or journalist"
      ]
     ],
     "graded": true,
     "expected": "athlete",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "athlete",
       "correct": true,
       "probabilities": {
        "athlete": 0.5239,
        "person": 0.3566,
        "organisation": 0.0219,
        "sports_manager": 0.0129,
        "organisation_member": 0.0107,
        "sports_team": 0.0102
       },
       "probability_of_expected": 0.5239,
       "proper_score": 0.8225
      },
      "Jev": {
       "answered": true,
       "pick": "athlete",
       "correct": true,
       "probabilities": {
        "athlete": 0.62,
        "sports_manager": 0.23,
        "coach": 0.1,
        "person": 0.03,
        "racing_driver": 0.01,
        "motorcycle_rider": 0.01
       },
       "probability_of_expected": 0.62,
       "proper_score": 0.8958
      },
      "JevFish": {
       "answered": true,
       "pick": "sports_manager",
       "correct": false,
       "probabilities": {
        "sports_manager": 0.3363,
        "athlete": 0.2406,
        "person": 0.0786,
        "coach": 0.0777,
        "racing_driver": 0.0555,
        "motorcycle_rider": 0.0368
       },
       "probability_of_expected": 0.2406,
       "proper_score": 0.6458
      },
      "NanoJev": {
       "answered": true,
       "pick": "organisation_member",
       "correct": false,
       "probabilities": {
        "organisation_member": 0.1797,
        "person": 0.1235,
        "organisation": 0.1098,
        "sports_manager": 0.0966,
        "artist": 0.0907,
        "sports_team": 0.0695,
        "athlete": 0.0643
       },
       "probability_of_expected": 0.0643,
       "proper_score": 0.5169
      },
      "OpenJev": {
       "answered": true,
       "pick": "athlete",
       "correct": true,
       "probabilities": {
        "athlete": 0.9947,
        "actor": 0.0013,
        "person": 0.0011,
        "artist": 0.0009,
        "wrestler": 0.0003,
        "organisation": 0.0002
       },
       "probability_of_expected": 0.9947,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "coach",
       "correct": false,
       "probabilities": {
        "coach": 0.1787,
        "athlete": 0.1089,
        "person": 0.1068,
        "sports_manager": 0.096,
        "organisation": 0.0731,
        "organisation_member": 0.0487
       },
       "probability_of_expected": 0.1089,
       "proper_score": 0.5677
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 5118.3,
    "Jev": 179.3,
    "JevFish": 2435.8,
    "NanoJev": 1658.8,
    "OpenJev": 377.9,
    "System One Scorer": 8831.2
   }
  },
  {
   "id": "data-and-operations:dbpedia-ccbffe43c144",
   "state": {
    "article_text": "The World Basketball Association, often abbreviated to the WBA, is a semi-professional men's spring basketball league in the United States. The league suspended operations after the 2013 season."
   },
   "state_tokens": 45,
   "tier": "common",
   "primary": "l1",
   "questions": [
    {
     "id": "l1",
     "type": "choice",
     "instructions": "Read state.article_text (the opening of a Wikipedia article). Which top-level DBpedia ontology class does the article's subject belong to?",
     "options": [
      [
       "agent",
       "A person, group, organisation, company, team, character or other acting entity"
      ],
      [
       "place",
       "A geographic or built location: settlement, building, natural feature, route, station, celestial body"
      ],
      [
       "species",
       "A biological organism: animal, plant, fungus or other eukaryote (incl. named breeds)"
      ],
      [
       "work",
       "A created work: album, song, book, periodical, software, comic, cartoon"
      ],
      [
       "event",
       "A dated happening: tournament, race, sports event, societal event, natural event, Olympics"
      ],
      [
       "sports_season",
       "One season of a sports team or league"
      ],
      [
       "unit_of_work",
       "A legal case or similar unit of proceedings"
      ],
      [
       "device",
       "A manufactured device such as an engine"
      ],
      [
       "topical_concept",
       "An abstract topic such as a genre"
      ]
     ],
     "graded": true,
     "expected": "agent",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.6396,
        "topical_concept": 0.1441,
        "event": 0.0849,
        "sports_season": 0.0849,
        "work": 0.0174,
        "place": 0.0107
       },
       "probability_of_expected": 0.6396,
       "proper_score": 0.9172
      },
      "Jev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.85,
        "event": 0.14,
        "sports_season": 0.01,
        "place": 0.0,
        "unit_of_work": 0.0,
        "topical_concept": 0.0
       },
       "probability_of_expected": 0.85,
       "proper_score": 0.9789
      },
      "JevFish": {
       "answered": true,
       "pick": "sports_season",
       "correct": false,
       "probabilities": {
        "sports_season": 0.9555,
        "event": 0.0348,
        "agent": 0.0046,
        "topical_concept": 0.0024,
        "place": 0.0007,
        "species": 0.0006
       },
       "probability_of_expected": 0.0046,
       "proper_score": 0.0475
      },
      "NanoJev": {
       "answered": true,
       "pick": "unit_of_work",
       "correct": false,
       "probabilities": {
        "unit_of_work": 0.2911,
        "agent": 0.2565,
        "sports_season": 0.2264,
        "topical_concept": 0.1383,
        "event": 0.0451,
        "place": 0.0248
       },
       "probability_of_expected": 0.2565,
       "proper_score": 0.6446
      },
      "OpenJev": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.5926,
        "event": 0.3307,
        "sports_season": 0.0714,
        "topical_concept": 0.0037,
        "place": 0.0012,
        "work": 0.0003
       },
       "probability_of_expected": 0.5926,
       "proper_score": 0.8597
      },
      "System One Scorer": {
       "answered": true,
       "pick": "agent",
       "correct": true,
       "probabilities": {
        "agent": 0.3728,
        "event": 0.2773,
        "sports_season": 0.0904,
        "work": 0.0731,
        "place": 0.0546,
        "topical_concept": 0.0471
       },
       "probability_of_expected": 0.3728,
       "proper_score": 0.7541
      }
     }
    },
    {
     "id": "l2",
     "type": "choice",
     "instructions": "Read state.article_text. Within the top-level class you chose, which second-level DBpedia class fits best? The options are the second-level classes of the article's true top-level class.",
     "options": [
      [
       "actor",
       "film, television or stage actor"
      ],
      [
       "artist",
       "visual or performing artist not covered by a more specific class"
      ],
      [
       "athlete",
       "sportsperson (generic, when no sport-specific class applies)"
      ],
      [
       "boxer",
       "boxer"
      ],
      [
       "british_royalty",
       "member of the British royal family"
      ],
      [
       "broadcaster",
       "radio or television station or network"
      ],
      [
       "cleric",
       "religious office holder"
      ],
      [
       "coach",
       "sports coach"
      ],
      [
       "company",
       "commercial company"
      ],
      [
       "educational_institution",
       "school, college or university"
      ],
      [
       "fictional_character",
       "fictional character (non-comics)"
      ],
      [
       "gridiron_football_player",
       "American or Canadian football player"
      ],
      [
       "group",
       "band or other group of people"
      ],
      [
       "motorcycle_rider",
       "motorcycle racer"
      ],
      [
       "organisation",
       "organisation not covered by a more specific class"
      ],
      [
       "organisation_member",
       "organisation member"
      ],
      [
       "person",
       "person not covered by a more specific class"
      ],
      [
       "politician",
       "politician or office holder"
      ],
      [
       "racing_driver",
       "motor racing driver"
      ],
      [
       "scientist",
       "scientist or academic"
      ],
      [
       "sports_league",
       "sports league"
      ],
      [
       "sports_manager",
       "sports manager"
      ],
      [
       "sports_team",
       "sports team or club"
      ],
      [
       "winter_sport_player",
       "winter-sport athlete (ice hockey, skiing, ...)"
      ],
      [
       "wrestler",
       "wrestler"
      ],
      [
       "writer",
       "author, poet or journalist"
      ]
     ],
     "graded": true,
     "expected": "sports_league",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "sports_league",
       "correct": true,
       "probabilities": {
        "sports_league": 0.8906,
        "organisation": 0.0886,
        "sports_team": 0.0073,
        "athlete": 0.0016,
        "group": 0.001,
        "company": 0.0008
       },
       "probability_of_expected": 0.8906,
       "proper_score": 0.9901
      },
      "Jev": {
       "answered": true,
       "pick": "sports_league",
       "correct": true,
       "probabilities": {
        "sports_league": 0.93,
        "organisation": 0.05,
        "sports_team": 0.01,
        "motorcycle_rider": 0.0,
        "coach": 0.0,
        "winter_sport_player": 0.0
       },
       "probability_of_expected": 0.93,
       "proper_score": 0.9962
      },
      "JevFish": {
       "answered": true,
       "pick": "sports_league",
       "correct": true,
       "probabilities": {
        "sports_league": 0.9523,
        "organisation": 0.0167,
        "sports_team": 0.0029,
        "person": 0.0021,
        "writer": 0.0019,
        "racing_driver": 0.0018
       },
       "probability_of_expected": 0.9523,
       "proper_score": 0.9987
      },
      "NanoJev": {
       "answered": true,
       "pick": "person",
       "correct": false,
       "probabilities": {
        "person": 0.195,
        "organisation": 0.1484,
        "artist": 0.1362,
        "athlete": 0.1116,
        "organisation_member": 0.0862,
        "sports_team": 0.081,
        "sports_league": 0.0661
       },
       "probability_of_expected": 0.0661,
       "proper_score": 0.5094
      },
      "OpenJev": {
       "answered": true,
       "pick": "sports_league",
       "correct": true,
       "probabilities": {
        "sports_league": 0.9568,
        "sports_team": 0.014,
        "person": 0.0084,
        "broadcaster": 0.0033,
        "actor": 0.0029,
        "scientist": 0.0018
       },
       "probability_of_expected": 0.9568,
       "proper_score": 0.9989
      },
      "System One Scorer": {
       "answered": true,
       "pick": "sports_league",
       "correct": true,
       "probabilities": {
        "sports_league": 0.6043,
        "organisation": 0.1906,
        "sports_team": 0.0368,
        "group": 0.0273,
        "organisation_member": 0.0206,
        "company": 0.0192
       },
       "probability_of_expected": 0.6043,
       "proper_score": 0.9017
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4539.4,
    "Jev": 153.0,
    "JevFish": 2480.2,
    "NanoJev": 1676.8,
    "OpenJev": 418.6,
    "System One Scorer": 9062.4
   }
  }
 ]
}