{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "abt-buy-product-match",
 "family": "retrieval-and-knowledge",
 "use_case": "entity-alignment",
 "vertical": "e-commerce",
 "dataset": "matchbench/Abt-Buy",
 "licence": "CC (variant unconfirmed on the source page) — FLAGGED",
 "provenance": "expert",
 "state_kind": "record",
 "primitive": "score",
 "items": 200,
 "scores": {
  "decider-2b": {
   "decisionScore": 82.7,
   "accuracy": 0.905,
   "answered": 200,
   "items": 200
  },
  "Jev": {
   "decisionScore": 94.91,
   "accuracy": 0.93,
   "answered": 200,
   "items": 200
  },
  "JevFish": {
   "decisionScore": 82.67,
   "accuracy": 0.885,
   "answered": 200,
   "items": 200
  },
  "NanoJev": {
   "decisionScore": 74.54,
   "accuracy": 0.665,
   "answered": 200,
   "items": 200
  },
  "OpenJev": {
   "decisionScore": 92.91,
   "accuracy": 0.92,
   "answered": 200,
   "items": 200
  },
  "System One Scorer": {
   "decisionScore": 85.37,
   "accuracy": 0.865,
   "answered": 200,
   "items": 200
  }
 },
 "samples": [
  {
   "id": "retrieval-and-knowledge:abtbuy-1024",
   "state": {
    "record_a": {
     "source": "abt",
     "name": "samsung 67 ' black widescreen series 7 led dlp hdtv hl67a750",
     "description": "samsung 67 ' black widescreen series 7 led dlp hdtv hl67a750 1920 x 1080p resolution 10,000:1 contrast ratio digital noise reduction cinemapure color engine get the full effect of 3d led backlight ntsc/atsc/clear qam built-in tuners 60,000 hours lamp life black finish",
     "price": null
    },
    "record_b": {
     "source": "buy",
     "name": "samsung hl72a650 72 ' widescreen 1080p dlp hdtv 2,500:1 contrast ratio",
     "description": "",
     "price": "2188.7"
    }
   },
   "state_tokens": 193,
   "tier": "common",
   "primary": "link_state",
   "questions": [
    {
     "id": "link_state",
     "type": "score",
     "instructions": "The state holds two product records, record_a from retailer abt and record_b from retailer buy (name, description, price). Rate how the two records relate as products.",
     "options": [
      [
       "0",
       "Two different products (different model or item, or only the same brand/category)."
      ],
      [
       "1",
       "Closely related products that may or may not be the same one: a colour or bundle variant, or a name that could refer to either."
      ],
      [
       "2",
       "One and the same product (same manufacturer model, possibly worded or abbreviated differently)."
      ]
     ],
     "graded": true,
     "expected": "level 0: Two different products (different model or item, or only the same brand/category).",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.5271,
        "1": 0.288,
        "2": 0.1849
       },
       "probability_of_expected": 0.5271,
       "proper_score": 0.8296
      },
      "Jev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.98,
        "1": 0.02,
        "2": 0.0
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9996
      },
      "JevFish": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.5918,
        "1": 0.2573,
        "2": 0.1509
       },
       "probability_of_expected": 0.5918,
       "proper_score": 0.8722
      },
      "NanoJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.4575,
        "2": 0.3728,
        "1": 0.1697
       },
       "probability_of_expected": 0.4575,
       "proper_score": 0.7689
      },
      "OpenJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.9908,
        "1": 0.0059,
        "2": 0.0033
       },
       "probability_of_expected": 0.9908,
       "proper_score": 0.9999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.6097,
        "1": 0.2414,
        "2": 0.1489
       },
       "probability_of_expected": 0.6097,
       "proper_score": 0.8836
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4672.9,
    "Jev": 151.6,
    "JevFish": 1492.4,
    "NanoJev": 656.8,
    "OpenJev": 380.6,
    "System One Scorer": 1664.1
   }
  },
  {
   "id": "retrieval-and-knowledge:abtbuy-1337",
   "state": {
    "record_a": {
     "source": "abt",
     "name": "garmin streetpilot c series vehicle suction mount black finish 0101060900",
     "description": "garmin streetpilot c series vehicle suction mount 0101060900 dashboard or friction mounting options are recommended except california 12/24 volt adapter",
     "price": null
    },
    "record_b": {
     "source": "buy",
     "name": "garmin suction cup mount/12-volt adapter kit 010-10979-00",
     "description": "navigator starter kit",
     "price": null
    }
   },
   "state_tokens": 125,
   "tier": "common",
   "primary": "link_state",
   "questions": [
    {
     "id": "link_state",
     "type": "score",
     "instructions": "The state holds two product records, record_a from retailer abt and record_b from retailer buy (name, description, price). Rate how the two records relate as products.",
     "options": [
      [
       "0",
       "Two different products (different model or item, or only the same brand/category)."
      ],
      [
       "1",
       "Closely related products that may or may not be the same one: a colour or bundle variant, or a name that could refer to either."
      ],
      [
       "2",
       "One and the same product (same manufacturer model, possibly worded or abbreviated differently)."
      ]
     ],
     "graded": true,
     "expected": "level 0: Two different products (different model or item, or only the same brand/category).",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "2",
       "correct": false,
       "probabilities": {
        "2": 0.4552,
        "0": 0.3199,
        "1": 0.2249
       },
       "probability_of_expected": 0.3199,
       "proper_score": 0.6398
      },
      "Jev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.51,
        "1": 0.44,
        "2": 0.05
       },
       "probability_of_expected": 0.51,
       "proper_score": 0.7819
      },
      "JevFish": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.6543,
        "1": 0.2377,
        "2": 0.108
       },
       "probability_of_expected": 0.6543,
       "proper_score": 0.9062
      },
      "NanoJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.5105,
        "2": 0.402,
        "1": 0.0875
       },
       "probability_of_expected": 0.5105,
       "proper_score": 0.7956
      },
      "OpenJev": {
       "answered": true,
       "pick": "1",
       "correct": false,
       "probabilities": {
        "1": 0.6213,
        "2": 0.2021,
        "0": 0.1766
       },
       "probability_of_expected": 0.1766,
       "proper_score": 0.4476
      },
      "System One Scorer": {
       "answered": true,
       "pick": "2",
       "correct": false,
       "probabilities": {
        "2": 0.6361,
        "1": 0.2636,
        "0": 0.1003
       },
       "probability_of_expected": 0.1003,
       "proper_score": 0.3582
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4061.8,
    "Jev": 154.9,
    "JevFish": 1511.7,
    "NanoJev": 727.1,
    "OpenJev": 425.4,
    "System One Scorer": 1449.3
   }
  },
  {
   "id": "retrieval-and-knowledge:abtbuy-1791",
   "state": {
    "record_a": {
     "source": "abt",
     "name": "sony black cyber-shot digital camera dscw150b",
     "description": "sony dscw150 black cyber-shot digital camera dscw150b 8.1 megapixels 2.7 ' lcd display 15 mb internal memory carl zeiss 5x optical zoom lens 16:9 high-resolution mode 9-point auto focus automatic macro shooting high sensitivity mode burst mode face detection intelligent scene recognition mode …",
     "price": null
    },
    "record_b": {
     "source": "buy",
     "name": "sony cyber-shot dsc-t300 digital camera black dsct300/b",
     "description": "10.1 megapixel 16:9 2x digital zoom 3.5 ' active matrix tft color lcd",
     "price": "273.95"
    }
   },
   "state_tokens": 179,
   "tier": "common",
   "primary": "link_state",
   "questions": [
    {
     "id": "link_state",
     "type": "score",
     "instructions": "The state holds two product records, record_a from retailer abt and record_b from retailer buy (name, description, price). Rate how the two records relate as products.",
     "options": [
      [
       "0",
       "Two different products (different model or item, or only the same brand/category)."
      ],
      [
       "1",
       "Closely related products that may or may not be the same one: a colour or bundle variant, or a name that could refer to either."
      ],
      [
       "2",
       "One and the same product (same manufacturer model, possibly worded or abbreviated differently)."
      ]
     ],
     "graded": true,
     "expected": "level 0: Two different products (different model or item, or only the same brand/category).",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.4564,
        "2": 0.2942,
        "1": 0.2494
       },
       "probability_of_expected": 0.4564,
       "proper_score": 0.7779
      },
      "Jev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 1.0,
        "1": 0.0,
        "2": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.587,
        "1": 0.2733,
        "2": 0.1397
       },
       "probability_of_expected": 0.587,
       "proper_score": 0.8676
      },
      "NanoJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.4458,
        "2": 0.3736,
        "1": 0.1806
       },
       "probability_of_expected": 0.4458,
       "proper_score": 0.7603
      },
      "OpenJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.9914,
        "1": 0.0045,
        "2": 0.0041
       },
       "probability_of_expected": 0.9914,
       "proper_score": 0.9999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.5286,
        "2": 0.2719,
        "1": 0.1995
       },
       "probability_of_expected": 0.5286,
       "proper_score": 0.832
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 5332.2,
    "Jev": 148.7,
    "JevFish": 1454.1,
    "NanoJev": 744.0,
    "OpenJev": 444.0,
    "System One Scorer": 1484.9
   }
  },
  {
   "id": "retrieval-and-knowledge:abtbuy-639",
   "state": {
    "record_a": {
     "source": "abt",
     "name": "lg 30 ' stainless steel freestanding gas range lrg30357ss",
     "description": "lg 30 ' stainless steel freestanding gas range lrg30357ss 5.0 cu . ft. capacity oven superboil burner 5 burner system smoothtouch controls warming drawer electronic ignition system one piece recessed cooktop self-cleaning oven stainless steel finish",
     "price": null
    },
    "record_b": {
     "source": "buy",
     "name": "lg 5.0 cu.ft . freestanding gas range",
     "description": "",
     "price": null
    }
   },
   "state_tokens": 127,
   "tier": "common",
   "primary": "link_state",
   "questions": [
    {
     "id": "link_state",
     "type": "score",
     "instructions": "The state holds two product records, record_a from retailer abt and record_b from retailer buy (name, description, price). Rate how the two records relate as products.",
     "options": [
      [
       "0",
       "Two different products (different model or item, or only the same brand/category)."
      ],
      [
       "1",
       "Closely related products that may or may not be the same one: a colour or bundle variant, or a name that could refer to either."
      ],
      [
       "2",
       "One and the same product (same manufacturer model, possibly worded or abbreviated differently)."
      ]
     ],
     "graded": true,
     "expected": "level 0: Two different products (different model or item, or only the same brand/category).",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "2",
       "correct": false,
       "probabilities": {
        "2": 0.5839,
        "0": 0.2225,
        "1": 0.1936
       },
       "probability_of_expected": 0.2225,
       "proper_score": 0.5085
      },
      "Jev": {
       "answered": true,
       "pick": "2",
       "correct": false,
       "probabilities": {
        "2": 0.77,
        "1": 0.22,
        "0": 0.01
       },
       "probability_of_expected": 0.01,
       "proper_score": 0.1893
      },
      "JevFish": {
       "answered": true,
       "pick": "2",
       "correct": false,
       "probabilities": {
        "2": 0.383,
        "0": 0.3667,
        "1": 0.2503
       },
       "probability_of_expected": 0.3667,
       "proper_score": 0.6948
      },
      "NanoJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.5092,
        "2": 0.3933,
        "1": 0.0975
       },
       "probability_of_expected": 0.5092,
       "proper_score": 0.7975
      },
      "OpenJev": {
       "answered": true,
       "pick": "2",
       "correct": false,
       "probabilities": {
        "2": 0.9751,
        "1": 0.0244,
        "0": 0.0005
       },
       "probability_of_expected": 0.0005,
       "proper_score": 0.0248
      },
      "System One Scorer": {
       "answered": true,
       "pick": "2",
       "correct": false,
       "probabilities": {
        "2": 0.7635,
        "1": 0.1574,
        "0": 0.0792
       },
       "probability_of_expected": 0.0792,
       "proper_score": 0.2722
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4010.2,
    "Jev": 224.6,
    "JevFish": 1517.5,
    "NanoJev": 770.3,
    "OpenJev": 378.0,
    "System One Scorer": 1414.2
   }
  },
  {
   "id": "retrieval-and-knowledge:abtbuy-680",
   "state": {
    "record_a": {
     "source": "abt",
     "name": "sirius sportster 4 with car kit sp4tk1",
     "description": "sirius sportster 4 with car kit sp4tk1 replays 44 minutes of content large easy-to-read 6-line display 30 channel presets game alerts and game zone features one touch jump",
     "price": null
    },
    "record_b": {
     "source": "buy",
     "name": "directed electronics sportster 4 sirius satellite radio receiver sp4tk1",
     "description": "xs fm transmitter 6 lines lcd",
     "price": "169.99"
    }
   },
   "state_tokens": 122,
   "tier": "common",
   "primary": "link_state",
   "questions": [
    {
     "id": "link_state",
     "type": "score",
     "instructions": "The state holds two product records, record_a from retailer abt and record_b from retailer buy (name, description, price). Rate how the two records relate as products.",
     "options": [
      [
       "0",
       "Two different products (different model or item, or only the same brand/category)."
      ],
      [
       "1",
       "Closely related products that may or may not be the same one: a colour or bundle variant, or a name that could refer to either."
      ],
      [
       "2",
       "One and the same product (same manufacturer model, possibly worded or abbreviated differently)."
      ]
     ],
     "graded": true,
     "expected": "level 2: One and the same product (same manufacturer model, possibly worded or abbreviated differently).",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "2",
       "correct": true,
       "probabilities": {
        "2": 0.6031,
        "1": 0.2191,
        "0": 0.1778
       },
       "probability_of_expected": 0.6031,
       "proper_score": 0.8814
      },
      "Jev": {
       "answered": true,
       "pick": "2",
       "correct": true,
       "probabilities": {
        "2": 0.91,
        "1": 0.08,
        "0": 0.01
       },
       "probability_of_expected": 0.91,
       "proper_score": 0.9927
      },
      "JevFish": {
       "answered": true,
       "pick": "2",
       "correct": true,
       "probabilities": {
        "2": 0.3664,
        "0": 0.3402,
        "1": 0.2934
       },
       "probability_of_expected": 0.3664,
       "proper_score": 0.6983
      },
      "NanoJev": {
       "answered": true,
       "pick": "0",
       "correct": false,
       "probabilities": {
        "0": 0.5416,
        "2": 0.373,
        "1": 0.0854
       },
       "probability_of_expected": 0.373,
       "proper_score": 0.6531
      },
      "OpenJev": {
       "answered": true,
       "pick": "2",
       "correct": true,
       "probabilities": {
        "2": 0.9919,
        "1": 0.0077,
        "0": 0.0004
       },
       "probability_of_expected": 0.9919,
       "proper_score": 0.9999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "2",
       "correct": true,
       "probabilities": {
        "2": 0.8565,
        "1": 0.1131,
        "0": 0.0304
       },
       "probability_of_expected": 0.8565,
       "proper_score": 0.9828
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4734.3,
    "Jev": 147.8,
    "JevFish": 1481.1,
    "NanoJev": 734.4,
    "OpenJev": 379.3,
    "System One Scorer": 1451.8
   }
  },
  {
   "id": "retrieval-and-knowledge:abtbuy-768",
   "state": {
    "record_a": {
     "source": "abt",
     "name": "samsung 32 ' series 6 lcd black flat panel hdtv ln32a650",
     "description": "samsung 32 ' series 6 lcd black flat panel hdtv ln32a650 full 1080p high definition resolution 16:9 widescreen aspect ratio 50,000:1 dynamic contrast ratio srs trusurround xt audio built-in digital tuner ( atsc/clear qam ) picture-in-picture auto volume leveler 4 hdmi slots swivel stand piano black …",
     "price": null
    },
    "record_b": {
     "source": "buy",
     "name": "samsung 4 series ln22a450 22 ' lcd tv",
     "description": "samsung ln22a450 22 ' widescreen 720p lcd hdtv 3000:1 dynamic contrast ratio 8ms response time piano black",
     "price": "397.77"
    }
   },
   "state_tokens": 215,
   "tier": "common",
   "primary": "link_state",
   "questions": [
    {
     "id": "link_state",
     "type": "score",
     "instructions": "The state holds two product records, record_a from retailer abt and record_b from retailer buy (name, description, price). Rate how the two records relate as products.",
     "options": [
      [
       "0",
       "Two different products (different model or item, or only the same brand/category)."
      ],
      [
       "1",
       "Closely related products that may or may not be the same one: a colour or bundle variant, or a name that could refer to either."
      ],
      [
       "2",
       "One and the same product (same manufacturer model, possibly worded or abbreviated differently)."
      ]
     ],
     "graded": true,
     "expected": "level 0: Two different products (different model or item, or only the same brand/category).",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.6144,
        "1": 0.2485,
        "2": 0.1371
       },
       "probability_of_expected": 0.6144,
       "proper_score": 0.8854
      },
      "Jev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 1.0,
        "1": 0.0,
        "2": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.7099,
        "1": 0.2054,
        "2": 0.0847
       },
       "probability_of_expected": 0.7099,
       "proper_score": 0.9332
      },
      "NanoJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.4425,
        "2": 0.3881,
        "1": 0.1694
       },
       "probability_of_expected": 0.4425,
       "proper_score": 0.755
      },
      "OpenJev": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.9935,
        "2": 0.0035,
        "1": 0.003
       },
       "probability_of_expected": 0.9935,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "0",
       "correct": true,
       "probabilities": {
        "0": 0.7517,
        "1": 0.1599,
        "2": 0.0884
       },
       "probability_of_expected": 0.7517,
       "proper_score": 0.9525
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4878.9,
    "Jev": 155.8,
    "JevFish": 1514.9,
    "NanoJev": 649.0,
    "OpenJev": 333.9,
    "System One Scorer": 1549.8
   }
  }
 ]
}