{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "cuad-checklist",
 "family": "safety-and-quality",
 "use_case": "compliance-verification",
 "vertical": "legal",
 "dataset": "nguha/legalbench",
 "licence": "CC-BY-4.0",
 "provenance": "expert",
 "state_kind": "record",
 "primitive": "noul",
 "items": 204,
 "scores": {
  "decider-2b": {
   "decisionScore": 95.1,
   "accuracy": 0.951,
   "answered": 204,
   "items": 204
  },
  "Jev": {
   "decisionScore": 91.61,
   "accuracy": 0.8922,
   "answered": 204,
   "items": 204
  },
  "JevFish": {
   "decisionScore": 86.35,
   "accuracy": 0.8088,
   "answered": 204,
   "items": 204
  },
  "NanoJev": {
   "decisionScore": 72.65,
   "accuracy": 0.4951,
   "answered": 204,
   "items": 204
  },
  "OpenJev": {
   "decisionScore": 91.22,
   "accuracy": 0.8922,
   "answered": 204,
   "items": 204
  },
  "System One Scorer": {
   "decisionScore": 92.1,
   "accuracy": 0.9265,
   "answered": 204,
   "items": 204
  }
 },
 "samples": [
  {
   "id": "safety-and-quality:cuad-checklist-anti-assignment-532",
   "state": {
    "document_id": "REGANHOLDINGCORP_03_31_2008-EX-10-LICENSE AND HOSTING AGREEMENT.PDF",
    "clause": "This Agreement shall be binding on the parties and their respective successors in interest and assigns, but neither party shall have the power to assign this Agreement without the prior written consent of the other party. LMG may not subcontract or delegate any of its duties or obligations of performance in this Agreement to any third party without the prior written consent of TAG.",
    "checklist_item": "Anti-assignment: requires consent before a party may assign the contract or its rights to a third party."
   },
   "state_tokens": 133,
   "tier": "common",
   "primary": "clause_matches_item",
   "questions": [
    {
     "id": "clause_matches_item",
     "type": "noul",
     "instructions": "The state holds one clause extracted from the contract named in document_id and one due-diligence checklist_item. Does the clause contain the kind of provision the checklist item describes?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.8969,
        "no": 0.1031
       },
       "probability_of_expected": 0.8969,
       "proper_score": 0.9894
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.99,
        "no": 0.01
       },
       "probability_of_expected": 0.99,
       "proper_score": 0.9999
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.713,
        "no": 0.287
       },
       "probability_of_expected": 0.713,
       "proper_score": 0.9176
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.6123,
        "no": 0.3877
       },
       "probability_of_expected": 0.6123,
       "proper_score": 0.8497
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9993,
        "no": 0.0007
       },
       "probability_of_expected": 0.9993,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.8765,
        "no": 0.1235
       },
       "probability_of_expected": 0.8765,
       "proper_score": 0.9847
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1220.7,
    "Jev": 145.5,
    "JevFish": 1462.2,
    "NanoJev": 639.3,
    "OpenJev": 378.0,
    "System One Scorer": 989.0
   }
  },
  {
   "id": "safety-and-quality:cuad-checklist-audit_rights-863",
   "state": {
    "document_id": "LiquidmetalTechnologiesInc_20200205_8-K_EX-10.1_11968198_EX-10.1_Development …",
    "clause": "New Application Technologies that have been developed by cooperation of the Parties shall be jointly and equally owned by the Parties, and any royalties for the use thereof by third parties shall be shared equitably between the Parties",
    "checklist_item": "Audit rights: lets a party inspect the other's books, records or premises to verify compliance."
   },
   "state_tokens": 113,
   "tier": "common",
   "primary": "clause_matches_item",
   "questions": [
    {
     "id": "clause_matches_item",
     "type": "noul",
     "instructions": "The state holds one clause extracted from the contract named in document_id and one due-diligence checklist_item. Does the clause contain the kind of provision the checklist item describes?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1912,
        "no": 0.8088
       },
       "probability_of_expected": 0.8088,
       "proper_score": 0.9634
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.02,
        "no": 0.98
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9996
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1622,
        "no": 0.8378
       },
       "probability_of_expected": 0.8378,
       "proper_score": 0.9737
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.6442,
        "no": 0.3558
       },
       "probability_of_expected": 0.3558,
       "proper_score": 0.585
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0,
        "no": 1.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.2291,
        "no": 0.7709
       },
       "probability_of_expected": 0.7709,
       "proper_score": 0.9475
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1226.3,
    "Jev": 217.0,
    "JevFish": 1603.6,
    "NanoJev": 624.3,
    "OpenJev": 294.5,
    "System One Scorer": 961.9
   }
  },
  {
   "id": "safety-and-quality:cuad-checklist-cap_on_liability-732",
   "state": {
    "document_id": "MANUFACTURERSSERVICESLTD_06_05_2000-EX-10.14-OUTSOURCING AGREEMENT.PDF",
    "clause": "This Agreement shall become effective on the Effective Date and shall continue for a period of three (3) years unless terminated as provided in Section 5.0.",
    "checklist_item": "Cap on liability: limits the amount a party can recover from the other for breach, loss or damages."
   },
   "state_tokens": 103,
   "tier": "common",
   "primary": "clause_matches_item",
   "questions": [
    {
     "id": "clause_matches_item",
     "type": "noul",
     "instructions": "The state holds one clause extracted from the contract named in document_id and one due-diligence checklist_item. Does the clause contain the kind of provision the checklist item describes?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1386,
        "no": 0.8614
       },
       "probability_of_expected": 0.8614,
       "proper_score": 0.9808
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.02,
        "no": 0.98
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9996
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1042,
        "no": 0.8958
       },
       "probability_of_expected": 0.8958,
       "proper_score": 0.9891
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.7491,
        "no": 0.2509
       },
       "probability_of_expected": 0.2509,
       "proper_score": 0.4389
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0001,
        "no": 0.9999
       },
       "probability_of_expected": 0.9999,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.2435,
        "no": 0.7565
       },
       "probability_of_expected": 0.7565,
       "proper_score": 0.9407
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1219.5,
    "Jev": 189.1,
    "JevFish": 1590.4,
    "NanoJev": 574.8,
    "OpenJev": 408.5,
    "System One Scorer": 1088.4
   }
  },
  {
   "id": "safety-and-quality:cuad-checklist-cap_on_liability-755",
   "state": {
    "document_id": "NEXSTARFINANCEHOLDINGSINC_03_27_2002-EX-10.26-OUTSOURCING AGREEMENT.PDF",
    "clause": "The Minimum BCF shall be Three Million Seven Hundred Thousand Dollars ($3,700,000) for calendar year 2002 and shall be increased on January 1, 2003 and on each January 1 thereafter in an amount equal to the percentage increase in the Consumer Price Index (published by the U.S. Department of Labor, Bureau of Labor Statistics, Philadelphia Regional Office - All Urban Consumers for the United States - All Items) (the \"PI\") over the prior year.",
    "checklist_item": "Cap on liability: limits the amount a party can recover from the other for breach, loss or damages."
   },
   "state_tokens": 177,
   "tier": "common",
   "primary": "clause_matches_item",
   "questions": [
    {
     "id": "clause_matches_item",
     "type": "noul",
     "instructions": "The state holds one clause extracted from the contract named in document_id and one due-diligence checklist_item. Does the clause contain the kind of provision the checklist item describes?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.3166,
        "no": 0.6834
       },
       "probability_of_expected": 0.6834,
       "proper_score": 0.8998
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.07,
        "no": 0.93
       },
       "probability_of_expected": 0.93,
       "proper_score": 0.9951
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1031,
        "no": 0.8969
       },
       "probability_of_expected": 0.8969,
       "proper_score": 0.9894
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": false,
       "probabilities": {
        "yes": 0.6169,
        "no": 0.3831
       },
       "probability_of_expected": 0.3831,
       "proper_score": 0.6194
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0001,
        "no": 0.9999
       },
       "probability_of_expected": 0.9999,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.3029,
        "no": 0.6971
       },
       "probability_of_expected": 0.6971,
       "proper_score": 0.9083
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1199.8,
    "Jev": 195.0,
    "JevFish": 1661.2,
    "NanoJev": 565.3,
    "OpenJev": 364.5,
    "System One Scorer": 1086.9
   }
  },
  {
   "id": "safety-and-quality:cuad-checklist-governing_law-168",
   "state": {
    "document_id": "MANUFACTURERSSERVICESLTD_06_05_2000-EX-10.14-OUTSOURCING AGREEMENT.PDF",
    "clause": "This Agreement and the rights and obligations of the parties hereto shall be construed in accordance with the substantive laws of the State of New York.",
    "checklist_item": "Governing law: names the state or country whose law governs the interpretation of the contract."
   },
   "state_tokens": 98,
   "tier": "common",
   "primary": "clause_matches_item",
   "questions": [
    {
     "id": "clause_matches_item",
     "type": "noul",
     "instructions": "The state holds one clause extracted from the contract named in document_id and one due-diligence checklist_item. Does the clause contain the kind of provision the checklist item describes?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.942,
        "no": 0.058
       },
       "probability_of_expected": 0.942,
       "proper_score": 0.9966
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.99,
        "no": 0.01
       },
       "probability_of_expected": 0.99,
       "proper_score": 0.9999
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7138,
        "no": 0.2862
       },
       "probability_of_expected": 0.7138,
       "proper_score": 0.9181
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7691,
        "no": 0.2309
       },
       "probability_of_expected": 0.7691,
       "proper_score": 0.9467
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9986,
        "no": 0.0014
       },
       "probability_of_expected": 0.9986,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.8786,
        "no": 0.1214
       },
       "probability_of_expected": 0.8786,
       "proper_score": 0.9853
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1045.7,
    "Jev": 187.2,
    "JevFish": 1728.4,
    "NanoJev": 557.0,
    "OpenJev": 365.2,
    "System One Scorer": 1147.1
   }
  },
  {
   "id": "safety-and-quality:cuad-checklist-non-compete-62",
   "state": {
    "document_id": "JOINTCORP_09_19_2014-EX-10.15-FRANCHISE AGREEMENT.PDF",
    "clause": "Upon expiration or termination of this Agreement for any reason, you agree not to engage in a competitive business for a period of two (2) years after the termination or expiration and within twenty-five (25) miles of your Franchise Premises or any other The Joint Corp. Location franchise location.",
    "checklist_item": "Non-compete: restricts a party from competing with the counterparty, or from operating in a named line of business, territory or market."
   },
   "state_tokens": 132,
   "tier": "common",
   "primary": "clause_matches_item",
   "questions": [
    {
     "id": "clause_matches_item",
     "type": "noul",
     "instructions": "The state holds one clause extracted from the contract named in document_id and one due-diligence checklist_item. Does the clause contain the kind of provision the checklist item describes?",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9446,
        "no": 0.0554
       },
       "probability_of_expected": 0.9446,
       "proper_score": 0.9969
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.99,
        "no": 0.01
       },
       "probability_of_expected": 0.99,
       "proper_score": 0.9999
      },
      "JevFish": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.7126,
        "no": 0.2874
       },
       "probability_of_expected": 0.7126,
       "proper_score": 0.9174
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.646,
        "no": 0.354
       },
       "probability_of_expected": 0.646,
       "proper_score": 0.8747
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9995,
        "no": 0.0005
       },
       "probability_of_expected": 0.9995,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.883,
        "no": 0.117
       },
       "probability_of_expected": 0.883,
       "proper_score": 0.9863
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1363.0,
    "Jev": 185.9,
    "JevFish": 1645.1,
    "NanoJev": 650.3,
    "OpenJev": 293.0,
    "System One Scorer": 1080.2
   }
  }
 ]
}