{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "unfair-tos-publish-gate",
 "family": "workflow-control",
 "use_case": "confidence-gated-actions",
 "vertical": "consumer-legal",
 "dataset": "coastalcph/lex_glue",
 "licence": "CC-BY-4.0",
 "provenance": "expert",
 "state_kind": "text",
 "primitive": "noul",
 "items": 198,
 "scores": {
  "decider-2b": {
   "decisionScore": 83.27,
   "accuracy": 0.7576,
   "answered": 198,
   "items": 198
  },
  "Jev": {
   "decisionScore": 87.1,
   "accuracy": 0.8434,
   "answered": 198,
   "items": 198
  },
  "JevFish": {
   "decisionScore": 79.68,
   "accuracy": 0.6717,
   "answered": 198,
   "items": 198
  },
  "NanoJev": {
   "decisionScore": 75.97,
   "accuracy": 0.6263,
   "answered": 198,
   "items": 198
  },
  "OpenJev": {
   "decisionScore": 83.9,
   "accuracy": 0.8182,
   "answered": 198,
   "items": 198
  },
  "System One Scorer": {
   "decisionScore": 83.92,
   "accuracy": 0.8232,
   "answered": 198,
   "items": 198
  }
 },
 "samples": [
  {
   "id": "workflow-control:tosgate-018f93aa6d06",
   "state": {
    "sentence": "if this specific paragraph is held unenforceable , then the entirety of this `` dispute resolution '' section will be deemed void ."
   },
   "state_tokens": 30,
   "tier": "common",
   "primary": "potentially_unfair",
   "questions": [
    {
     "id": "potentially_unfair",
     "type": "noul",
     "instructions": "Read `sentence`, one sentence from an online platform's Terms of Service (text is lower-cased and tokenised). Answer yes if the sentence is a potentially unfair contractual term for consumers under EU consumer law: it limits the provider's liability, lets the provider terminate the contract unilaterally, lets the provider change the terms unilaterally, lets the provider remove user content at its discretion, binds the user merely by use of the service, imposes a choice of governing law, imposes a jurisdiction for disputes, or imposes arbitration. Answer no for any other sentence, including neutral descriptions of the service and sentences about the user's own obligations.",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.464,
        "no": 0.536
       },
       "probability_of_expected": 0.536,
       "proper_score": 0.7847
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.35,
        "no": 0.65
       },
       "probability_of_expected": 0.65,
       "proper_score": 0.8775
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.4264,
        "no": 0.5736
       },
       "probability_of_expected": 0.5736,
       "proper_score": 0.8182
      },
      "NanoJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.3495,
        "no": 0.6505
       },
       "probability_of_expected": 0.6505,
       "proper_score": 0.8778
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0247,
        "no": 0.9753
       },
       "probability_of_expected": 0.9753,
       "proper_score": 0.9994
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.4743,
        "no": 0.5257
       },
       "probability_of_expected": 0.5257,
       "proper_score": 0.7751
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 4597.7,
    "Jev": 139.1,
    "JevFish": 1886.2,
    "NanoJev": 754.1,
    "OpenJev": 424.6,
    "System One Scorer": 1788.3
   }
  },
  {
   "id": "workflow-control:tosgate-123eac3c8c49",
   "state": {
    "sentence": "we may change our seller fees from time to time by posting the changes on the ebay site 14 days in advance , but with no advance notice required for temporary promotions or any changes that result in the reduction of fees ."
   },
   "state_tokens": 49,
   "tier": "common",
   "primary": "potentially_unfair",
   "questions": [
    {
     "id": "potentially_unfair",
     "type": "noul",
     "instructions": "Read `sentence`, one sentence from an online platform's Terms of Service (text is lower-cased and tokenised). Answer yes if the sentence is a potentially unfair contractual term for consumers under EU consumer law: it limits the provider's liability, lets the provider terminate the contract unilaterally, lets the provider change the terms unilaterally, lets the provider remove user content at its discretion, binds the user merely by use of the service, imposes a choice of governing law, imposes a jurisdiction for disputes, or imposes arbitration. Answer no for any other sentence, including neutral descriptions of the service and sentences about the user's own obligations.",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": false,
       "probabilities": {
        "yes": 0.2311,
        "no": 0.7689
       },
       "probability_of_expected": 0.2311,
       "proper_score": 0.4088
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.88,
        "no": 0.12
       },
       "probability_of_expected": 0.88,
       "proper_score": 0.9856
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": false,
       "probabilities": {
        "yes": 0.1865,
        "no": 0.8135
       },
       "probability_of_expected": 0.1865,
       "proper_score": 0.3382
      },
      "NanoJev": {
       "answered": true,
       "pick": "no",
       "correct": false,
       "probabilities": {
        "yes": 0.4654,
        "no": 0.5346
       },
       "probability_of_expected": 0.4654,
       "proper_score": 0.7142
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9944,
        "no": 0.0056
       },
       "probability_of_expected": 0.9944,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.6994,
        "no": 0.3006
       },
       "probability_of_expected": 0.6994,
       "proper_score": 0.9096
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1452.8,
    "Jev": 198.5,
    "JevFish": 1557.8,
    "NanoJev": 627.3,
    "OpenJev": 385.1,
    "System One Scorer": 1121.7
   }
  },
  {
   "id": "workflow-control:tosgate-710c7d661bd1",
   "state": {
    "sentence": "please review our commercial terms for information about additional contract terms relating to such purchases ."
   },
   "state_tokens": 20,
   "tier": "common",
   "primary": "potentially_unfair",
   "questions": [
    {
     "id": "potentially_unfair",
     "type": "noul",
     "instructions": "Read `sentence`, one sentence from an online platform's Terms of Service (text is lower-cased and tokenised). Answer yes if the sentence is a potentially unfair contractual term for consumers under EU consumer law: it limits the provider's liability, lets the provider terminate the contract unilaterally, lets the provider change the terms unilaterally, lets the provider remove user content at its discretion, binds the user merely by use of the service, imposes a choice of governing law, imposes a jurisdiction for disputes, or imposes arbitration. Answer no for any other sentence, including neutral descriptions of the service and sentences about the user's own obligations.",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0182,
        "no": 0.9818
       },
       "probability_of_expected": 0.9818,
       "proper_score": 0.9997
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.06,
        "no": 0.94
       },
       "probability_of_expected": 0.94,
       "proper_score": 0.9964
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1807,
        "no": 0.8193
       },
       "probability_of_expected": 0.8193,
       "proper_score": 0.9673
      },
      "NanoJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.441,
        "no": 0.559
       },
       "probability_of_expected": 0.559,
       "proper_score": 0.8056
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0005,
        "no": 0.9995
       },
       "probability_of_expected": 0.9995,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.3434,
        "no": 0.6566
       },
       "probability_of_expected": 0.6566,
       "proper_score": 0.8821
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1842.0,
    "Jev": 153.1,
    "JevFish": 1645.4,
    "NanoJev": 623.2,
    "OpenJev": 363.7,
    "System One Scorer": 1060.4
   }
  },
  {
   "id": "workflow-control:tosgate-af8e0d59f4ed",
   "state": {
    "sentence": "in the event that duolingo suspends or terminates your use of the service or these terms and conditions or you close your account voluntarily , you understand and agree that you will receive no refund or exchange of any kind , including for any unused virtual currency or other virtual item , any content or data associated with your use of the service , or for anything else ."
   },
   "state_tokens": 76,
   "tier": "common",
   "primary": "potentially_unfair",
   "questions": [
    {
     "id": "potentially_unfair",
     "type": "noul",
     "instructions": "Read `sentence`, one sentence from an online platform's Terms of Service (text is lower-cased and tokenised). Answer yes if the sentence is a potentially unfair contractual term for consumers under EU consumer law: it limits the provider's liability, lets the provider terminate the contract unilaterally, lets the provider change the terms unilaterally, lets the provider remove user content at its discretion, binds the user merely by use of the service, imposes a choice of governing law, imposes a jurisdiction for disputes, or imposes arbitration. Answer no for any other sentence, including neutral descriptions of the service and sentences about the user's own obligations.",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.5716,
        "no": 0.4284
       },
       "probability_of_expected": 0.5716,
       "proper_score": 0.8165
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.88,
        "no": 0.12
       },
       "probability_of_expected": 0.88,
       "proper_score": 0.9856
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": false,
       "probabilities": {
        "yes": 0.4996,
        "no": 0.5004
       },
       "probability_of_expected": 0.4996,
       "proper_score": 0.7496
      },
      "NanoJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.5913,
        "no": 0.4087
       },
       "probability_of_expected": 0.5913,
       "proper_score": 0.8329
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9902,
        "no": 0.0098
       },
       "probability_of_expected": 0.9902,
       "proper_score": 0.9999
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.6143,
        "no": 0.3857
       },
       "probability_of_expected": 0.6143,
       "proper_score": 0.8512
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1638.0,
    "Jev": 200.2,
    "JevFish": 1535.3,
    "NanoJev": 524.7,
    "OpenJev": 303.3,
    "System One Scorer": 1213.0
   }
  },
  {
   "id": "workflow-control:tosgate-c34f00fee30d",
   "state": {
    "sentence": "you also agree not to : circumvent , remove , alter , deactivate , degrade or thwart any of the content protections in the netflix service ; use any robot , spider , scraper or other automated means to access the netflix service ; decompile , reverse engineer or disassemble any software or other products or processes accessible through the netflix service ; insert any code or product or manipulate the content of the netflix service in any way ; or use any data mining , data gathering or extraction method ."
   },
   "state_tokens": 103,
   "tier": "common",
   "primary": "potentially_unfair",
   "questions": [
    {
     "id": "potentially_unfair",
     "type": "noul",
     "instructions": "Read `sentence`, one sentence from an online platform's Terms of Service (text is lower-cased and tokenised). Answer yes if the sentence is a potentially unfair contractual term for consumers under EU consumer law: it limits the provider's liability, lets the provider terminate the contract unilaterally, lets the provider change the terms unilaterally, lets the provider remove user content at its discretion, binds the user merely by use of the service, imposes a choice of governing law, imposes a jurisdiction for disputes, or imposes arbitration. Answer no for any other sentence, including neutral descriptions of the service and sentences about the user's own obligations.",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "no",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.1505,
        "no": 0.8495
       },
       "probability_of_expected": 0.8495,
       "proper_score": 0.9773
      },
      "Jev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.06,
        "no": 0.94
       },
       "probability_of_expected": 0.94,
       "proper_score": 0.9964
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.4451,
        "no": 0.5549
       },
       "probability_of_expected": 0.5549,
       "proper_score": 0.8019
      },
      "NanoJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.455,
        "no": 0.545
       },
       "probability_of_expected": 0.545,
       "proper_score": 0.793
      },
      "OpenJev": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.0003,
        "no": 0.9997
       },
       "probability_of_expected": 0.9997,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "no",
       "correct": true,
       "probabilities": {
        "yes": 0.2273,
        "no": 0.7727
       },
       "probability_of_expected": 0.7727,
       "proper_score": 0.9483
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1351.6,
    "Jev": 145.5,
    "JevFish": 1578.9,
    "NanoJev": 692.7,
    "OpenJev": 359.6,
    "System One Scorer": 1241.2
   }
  },
  {
   "id": "workflow-control:tosgate-f3dd463d69ac",
   "state": {
    "sentence": "by continuing to access or use the site or services after we have posted a modification on the site or have provided you with notice of a modification , you are indicating that you agree to be bound by the modified terms ."
   },
   "state_tokens": 47,
   "tier": "common",
   "primary": "potentially_unfair",
   "questions": [
    {
     "id": "potentially_unfair",
     "type": "noul",
     "instructions": "Read `sentence`, one sentence from an online platform's Terms of Service (text is lower-cased and tokenised). Answer yes if the sentence is a potentially unfair contractual term for consumers under EU consumer law: it limits the provider's liability, lets the provider terminate the contract unilaterally, lets the provider change the terms unilaterally, lets the provider remove user content at its discretion, binds the user merely by use of the service, imposes a choice of governing law, imposes a jurisdiction for disputes, or imposes arbitration. Answer no for any other sentence, including neutral descriptions of the service and sentences about the user's own obligations.",
     "options": [
      [
       "yes",
       ""
      ],
      [
       "no",
       ""
      ]
     ],
     "graded": true,
     "expected": "yes",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "no",
       "correct": false,
       "probabilities": {
        "yes": 0.2863,
        "no": 0.7137
       },
       "probability_of_expected": 0.2863,
       "proper_score": 0.4906
      },
      "Jev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.94,
        "no": 0.06
       },
       "probability_of_expected": 0.94,
       "proper_score": 0.9964
      },
      "JevFish": {
       "answered": true,
       "pick": "no",
       "correct": false,
       "probabilities": {
        "yes": 0.4083,
        "no": 0.5917
       },
       "probability_of_expected": 0.4083,
       "proper_score": 0.6499
      },
      "NanoJev": {
       "answered": true,
       "pick": "no",
       "correct": false,
       "probabilities": {
        "yes": 0.4853,
        "no": 0.5147
       },
       "probability_of_expected": 0.4853,
       "proper_score": 0.7351
      },
      "OpenJev": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.9988,
        "no": 0.0012
       },
       "probability_of_expected": 0.9988,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "yes",
       "correct": true,
       "probabilities": {
        "yes": 0.6384,
        "no": 0.3616
       },
       "probability_of_expected": 0.6384,
       "proper_score": 0.8693
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 1404.6,
    "Jev": 174.7,
    "JevFish": 1604.3,
    "NanoJev": 630.3,
    "OpenJev": 289.8,
    "System One Scorer": 1151.9
   }
  }
 ]
}