{
 "schemaVersion": "blobfish.typed-decision-bench-task.v1",
 "version": "0.3.0",
 "task": "cfpb-complaint-product-queue",
 "family": "workflow-control",
 "use_case": "support-inbox-triage",
 "vertical": "finance",
 "dataset": "liri-uzh/cfpb-complaints-mini",
 "licence": "CC0-1.0 (CFPB public domain)",
 "provenance": "outcome",
 "state_kind": "record",
 "primitive": "choice",
 "items": 199,
 "scores": {
  "decider-2b": {
   "decisionScore": 79.71,
   "accuracy": 0.7286,
   "answered": 199,
   "items": 199
  },
  "Jev": {
   "decisionScore": 83.45,
   "accuracy": 0.794,
   "answered": 199,
   "items": 199
  },
  "JevFish": {
   "decisionScore": 73.61,
   "accuracy": 0.6432,
   "answered": 199,
   "items": 199
  },
  "NanoJev": {
   "decisionScore": 63.41,
   "accuracy": 0.4372,
   "answered": 199,
   "items": 199
  },
  "OpenJev": {
   "decisionScore": 78.94,
   "accuracy": 0.7688,
   "answered": 199,
   "items": 199
  },
  "System One Scorer": {
   "decisionScore": 82.62,
   "accuracy": 0.7789,
   "answered": 199,
   "items": 199
  }
 },
 "samples": [
  {
   "id": "workflow-control:cfpb-6c8cf595e170",
   "state": {
    "complaint": "It started by an email displaying PayPal 's legitimate website and there was an attempt from an unknown source trying to withdraw XXXX in bio coins from my account invoice # XXXX. They applied me with Paypal 's toll free phone number XXXX, that's why I responded. Once I called, an agent by the name of XXXX who had an XXXX XXXX  answered and told me that he was going to walk me through to have this charge reverse. He had me go to this website XXXX because the other ones he gave me wasn't working. Once I downloaded the website he said that he had access to my account, this appeared to be a sharing process, and had me type this email XXXX and to send $ XXXX to it. This is the transaction number XXXX, A name of XXXX XXXX appeared and the transaction went through not knowing he was leading me to the XXXX. I felt uneasy once it was over. What …"
   },
   "state_tokens": 201,
   "tier": "common",
   "primary": "product_queue",
   "questions": [
    {
     "id": "product_queue",
     "type": "choice",
     "instructions": "Read the `complaint` narrative. Choose the single financial product line the consumer is complaining about, i.e. the specialist queue that should own this ticket. Personal details are redacted as XXXX.",
     "options": [
      [
       "bank_account",
       "Checking or savings account, or bank account services (deposits, overdrafts, holds, closures, ATM/debit issues)"
      ],
      [
       "credit_card",
       "Credit card or prepaid card (charges, rewards, card fees, card fraud, card billing disputes)"
      ],
      [
       "credit_reporting",
       "Credit reporting agencies, credit repair, or other consumer reports (incorrect report data, disputes with bureaus, identity-theft entries)"
      ],
      [
       "debt_collection",
       "A third-party or first-party debt collector's conduct or a debt the consumer says is not theirs"
      ],
      [
       "money_transfer",
       "Money transfer, virtual currency, or money services (wires, P2P apps, remittances, crypto, check cashing)"
      ],
      [
       "mortgage",
       "Home mortgage loans (origination, servicing, escrow, modification, foreclosure)"
      ],
      [
       "student_loan",
       "Federal or private student loans (servicing, repayment plans, forgiveness, collections on student debt)"
      ]
     ],
     "graded": true,
     "expected": "money_transfer",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.9081,
        "credit_card": 0.0615,
        "bank_account": 0.0176,
        "debt_collection": 0.0067,
        "mortgage": 0.0022,
        "credit_reporting": 0.0021
       },
       "probability_of_expected": 0.9081,
       "proper_score": 0.9937
      },
      "Jev": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.98,
        "credit_card": 0.02,
        "bank_account": 0.0,
        "debt_collection": 0.0,
        "student_loan": 0.0,
        "mortgage": 0.0
       },
       "probability_of_expected": 0.98,
       "proper_score": 0.9996
      },
      "JevFish": {
       "answered": true,
       "pick": "debt_collection",
       "correct": false,
       "probabilities": {
        "debt_collection": 0.7613,
        "money_transfer": 0.1146,
        "bank_account": 0.0625,
        "credit_card": 0.0312,
        "credit_reporting": 0.0233,
        "student_loan": 0.004
       },
       "probability_of_expected": 0.1146,
       "proper_score": 0.3155
      },
      "NanoJev": {
       "answered": true,
       "pick": "debt_collection",
       "correct": false,
       "probabilities": {
        "debt_collection": 0.38,
        "money_transfer": 0.2444,
        "credit_reporting": 0.1553,
        "credit_card": 0.1168,
        "bank_account": 0.0468,
        "student_loan": 0.0387
       },
       "probability_of_expected": 0.2444,
       "proper_score": 0.6214
      },
      "OpenJev": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.9991,
        "credit_card": 0.0004,
        "bank_account": 0.0004,
        "debt_collection": 0.0001,
        "mortgage": 0.0,
        "credit_reporting": 0.0
       },
       "probability_of_expected": 0.9991,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.4522,
        "debt_collection": 0.3174,
        "credit_card": 0.0648,
        "bank_account": 0.0639,
        "student_loan": 0.0386,
        "credit_reporting": 0.0324
       },
       "probability_of_expected": 0.4522,
       "proper_score": 0.7937
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2873.0,
    "Jev": 244.9,
    "JevFish": 1566.4,
    "NanoJev": 681.7,
    "OpenJev": 354.2,
    "System One Scorer": 2607.7
   }
  },
  {
   "id": "workflow-control:cfpb-8e41c03b14a3",
   "state": {
    "complaint": "- Our son purchased a {$50.00} gift card for us as a Christmas Gift. ( XX/XX/XXXX ) - The first time we activated it was on XX/XX/XXXX. \n\n- Immediately upon activation, 7 very similar charges popped up, all for just a few dollars, and all within a few cents of each other. ( {$3.00}, {$3.00}, {$3.00}, {$3.00}, {$3.00}, {$3.00}, {$3.00} ) We immediately called customer service to report this, but we were told that they could not officially put in a dispute until 1 month later ( end of cycle?? ). \n\n- On XX/XX/XXXX, we again called, and an \" official '' dispute was entered. \n\nAlso at this time, our account STILL read these charges as PENDING, which seems to indicate that they had not yet been processed ( much like a credit card ). \n\n- Our card was frozen pending the investigation. \n\n- We repeatedly called, and got no results until …"
   },
   "state_tokens": 244,
   "tier": "common",
   "primary": "product_queue",
   "questions": [
    {
     "id": "product_queue",
     "type": "choice",
     "instructions": "Read the `complaint` narrative. Choose the single financial product line the consumer is complaining about, i.e. the specialist queue that should own this ticket. Personal details are redacted as XXXX.",
     "options": [
      [
       "bank_account",
       "Checking or savings account, or bank account services (deposits, overdrafts, holds, closures, ATM/debit issues)"
      ],
      [
       "credit_card",
       "Credit card or prepaid card (charges, rewards, card fees, card fraud, card billing disputes)"
      ],
      [
       "credit_reporting",
       "Credit reporting agencies, credit repair, or other consumer reports (incorrect report data, disputes with bureaus, identity-theft entries)"
      ],
      [
       "debt_collection",
       "A third-party or first-party debt collector's conduct or a debt the consumer says is not theirs"
      ],
      [
       "money_transfer",
       "Money transfer, virtual currency, or money services (wires, P2P apps, remittances, crypto, check cashing)"
      ],
      [
       "mortgage",
       "Home mortgage loans (origination, servicing, escrow, modification, foreclosure)"
      ],
      [
       "student_loan",
       "Federal or private student loans (servicing, repayment plans, forgiveness, collections on student debt)"
      ]
     ],
     "graded": true,
     "expected": "credit_card",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "credit_card",
       "correct": true,
       "probabilities": {
        "credit_card": 0.997,
        "bank_account": 0.0008,
        "credit_reporting": 0.0007,
        "money_transfer": 0.0005,
        "debt_collection": 0.0004,
        "mortgage": 0.0003
       },
       "probability_of_expected": 0.997,
       "proper_score": 1.0
      },
      "Jev": {
       "answered": true,
       "pick": "credit_card",
       "correct": true,
       "probabilities": {
        "credit_card": 1.0,
        "bank_account": 0.0,
        "debt_collection": 0.0,
        "money_transfer": 0.0,
        "student_loan": 0.0,
        "mortgage": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "credit_card",
       "correct": true,
       "probabilities": {
        "credit_card": 0.9668,
        "credit_reporting": 0.0186,
        "bank_account": 0.0052,
        "debt_collection": 0.005,
        "student_loan": 0.0024,
        "mortgage": 0.0015
       },
       "probability_of_expected": 0.9668,
       "proper_score": 0.9992
      },
      "NanoJev": {
       "answered": true,
       "pick": "credit_card",
       "correct": true,
       "probabilities": {
        "credit_card": 0.5085,
        "credit_reporting": 0.2257,
        "money_transfer": 0.1213,
        "debt_collection": 0.1078,
        "bank_account": 0.0183,
        "student_loan": 0.0127
       },
       "probability_of_expected": 0.5085,
       "proper_score": 0.8403
      },
      "OpenJev": {
       "answered": true,
       "pick": "credit_card",
       "correct": true,
       "probabilities": {
        "credit_card": 0.9727,
        "bank_account": 0.0244,
        "money_transfer": 0.0017,
        "debt_collection": 0.0009,
        "mortgage": 0.0001,
        "credit_reporting": 0.0001
       },
       "probability_of_expected": 0.9727,
       "proper_score": 0.9993
      },
      "System One Scorer": {
       "answered": true,
       "pick": "credit_card",
       "correct": true,
       "probabilities": {
        "credit_card": 0.4061,
        "debt_collection": 0.2378,
        "money_transfer": 0.1606,
        "bank_account": 0.0976,
        "credit_reporting": 0.0375,
        "mortgage": 0.0346
       },
       "probability_of_expected": 0.4061,
       "proper_score": 0.7761
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2335.6,
    "Jev": 158.5,
    "JevFish": 1580.7,
    "NanoJev": 660.9,
    "OpenJev": 348.7,
    "System One Scorer": 2622.1
   }
  },
  {
   "id": "workflow-control:cfpb-958e98746a3f",
   "state": {
    "complaint": "On XX/XX/XXXX and XXXX a fraudulent scammer requested funds {$50.00}, {$130.00} from my account. I declined the payment. After decline these payments to this fraudulent merchant there were two duplicate unauthorized payments two this merchant that were approved in error of the system. Cash App will not reverse these funds after multiple chats, phone call, and direct dispute."
   },
   "state_tokens": 88,
   "tier": "common",
   "primary": "product_queue",
   "questions": [
    {
     "id": "product_queue",
     "type": "choice",
     "instructions": "Read the `complaint` narrative. Choose the single financial product line the consumer is complaining about, i.e. the specialist queue that should own this ticket. Personal details are redacted as XXXX.",
     "options": [
      [
       "bank_account",
       "Checking or savings account, or bank account services (deposits, overdrafts, holds, closures, ATM/debit issues)"
      ],
      [
       "credit_card",
       "Credit card or prepaid card (charges, rewards, card fees, card fraud, card billing disputes)"
      ],
      [
       "credit_reporting",
       "Credit reporting agencies, credit repair, or other consumer reports (incorrect report data, disputes with bureaus, identity-theft entries)"
      ],
      [
       "debt_collection",
       "A third-party or first-party debt collector's conduct or a debt the consumer says is not theirs"
      ],
      [
       "money_transfer",
       "Money transfer, virtual currency, or money services (wires, P2P apps, remittances, crypto, check cashing)"
      ],
      [
       "mortgage",
       "Home mortgage loans (origination, servicing, escrow, modification, foreclosure)"
      ],
      [
       "student_loan",
       "Federal or private student loans (servicing, repayment plans, forgiveness, collections on student debt)"
      ]
     ],
     "graded": true,
     "expected": "money_transfer",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.8849,
        "bank_account": 0.08,
        "credit_card": 0.0306,
        "credit_reporting": 0.0014,
        "debt_collection": 0.0011,
        "mortgage": 0.0011
       },
       "probability_of_expected": 0.8849,
       "proper_score": 0.9897
      },
      "Jev": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 1.0,
        "student_loan": 0.0,
        "credit_card": 0.0,
        "mortgage": 0.0,
        "debt_collection": 0.0,
        "credit_reporting": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "credit_card",
       "correct": false,
       "probabilities": {
        "credit_card": 0.6787,
        "bank_account": 0.1347,
        "credit_reporting": 0.0966,
        "money_transfer": 0.0471,
        "debt_collection": 0.0376,
        "student_loan": 0.0033
       },
       "probability_of_expected": 0.0471,
       "proper_score": 0.3013
      },
      "NanoJev": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.3035,
        "debt_collection": 0.242,
        "credit_card": 0.216,
        "credit_reporting": 0.1743,
        "bank_account": 0.0345,
        "student_loan": 0.0211
       },
       "probability_of_expected": 0.3035,
       "proper_score": 0.6888
      },
      "OpenJev": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.9996,
        "bank_account": 0.0002,
        "credit_card": 0.0001,
        "debt_collection": 0.0,
        "mortgage": 0.0,
        "student_loan": 0.0
       },
       "probability_of_expected": 0.9996,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "money_transfer",
       "correct": true,
       "probabilities": {
        "money_transfer": 0.8799,
        "debt_collection": 0.0457,
        "credit_card": 0.03,
        "bank_account": 0.0266,
        "mortgage": 0.0065,
        "credit_reporting": 0.0062
       },
       "probability_of_expected": 0.8799,
       "proper_score": 0.9909
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2025.3,
    "Jev": 206.5,
    "JevFish": 1454.6,
    "NanoJev": 748.5,
    "OpenJev": 313.9,
    "System One Scorer": 2912.2
   }
  },
  {
   "id": "workflow-control:cfpb-9cdc26fc2a69",
   "state": {
    "complaint": "I trust this letter finds you well. I am writing to bring to your attention several discrepancies in my credit report that are in violation of my federally protected consumer rights under the Fair Credit Reporting Act ( FCRA ), specifically USC 1681. \n\nList of Accounts in Question : XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX  According to the provisions outlined in the FCRA : 15 U.S.C 1681 Section 602 A : Affirms my right to privacy.\n\n15 U.S.C 1681 Section 604 Section 2 : States that a consumer reporting agency can not furnish an account without my explicit written instructions.\n\n15 U.S.C 1681 ( a ) ( 5 ) …"
   },
   "state_tokens": 270,
   "tier": "common",
   "primary": "product_queue",
   "questions": [
    {
     "id": "product_queue",
     "type": "choice",
     "instructions": "Read the `complaint` narrative. Choose the single financial product line the consumer is complaining about, i.e. the specialist queue that should own this ticket. Personal details are redacted as XXXX.",
     "options": [
      [
       "bank_account",
       "Checking or savings account, or bank account services (deposits, overdrafts, holds, closures, ATM/debit issues)"
      ],
      [
       "credit_card",
       "Credit card or prepaid card (charges, rewards, card fees, card fraud, card billing disputes)"
      ],
      [
       "credit_reporting",
       "Credit reporting agencies, credit repair, or other consumer reports (incorrect report data, disputes with bureaus, identity-theft entries)"
      ],
      [
       "debt_collection",
       "A third-party or first-party debt collector's conduct or a debt the consumer says is not theirs"
      ],
      [
       "money_transfer",
       "Money transfer, virtual currency, or money services (wires, P2P apps, remittances, crypto, check cashing)"
      ],
      [
       "mortgage",
       "Home mortgage loans (origination, servicing, escrow, modification, foreclosure)"
      ],
      [
       "student_loan",
       "Federal or private student loans (servicing, repayment plans, forgiveness, collections on student debt)"
      ]
     ],
     "graded": true,
     "expected": "credit_reporting",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "credit_reporting",
       "correct": true,
       "probabilities": {
        "credit_reporting": 0.9932,
        "credit_card": 0.0046,
        "bank_account": 0.0006,
        "debt_collection": 0.0006,
        "mortgage": 0.0005,
        "student_loan": 0.0004
       },
       "probability_of_expected": 0.9932,
       "proper_score": 1.0
      },
      "Jev": {
       "answered": true,
       "pick": "credit_reporting",
       "correct": true,
       "probabilities": {
        "credit_reporting": 1.0,
        "bank_account": 0.0,
        "debt_collection": 0.0,
        "student_loan": 0.0,
        "money_transfer": 0.0,
        "mortgage": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "credit_reporting",
       "correct": true,
       "probabilities": {
        "credit_reporting": 0.989,
        "credit_card": 0.0047,
        "debt_collection": 0.0026,
        "student_loan": 0.0017,
        "bank_account": 0.0013,
        "mortgage": 0.0005
       },
       "probability_of_expected": 0.989,
       "proper_score": 0.9999
      },
      "NanoJev": {
       "answered": true,
       "pick": "credit_reporting",
       "correct": true,
       "probabilities": {
        "credit_reporting": 0.2972,
        "debt_collection": 0.2323,
        "credit_card": 0.1609,
        "bank_account": 0.1146,
        "money_transfer": 0.098,
        "student_loan": 0.08
       },
       "probability_of_expected": 0.2972,
       "proper_score": 0.6984
      },
      "OpenJev": {
       "answered": true,
       "pick": "credit_reporting",
       "correct": true,
       "probabilities": {
        "credit_reporting": 0.9999,
        "debt_collection": 0.0,
        "credit_card": 0.0,
        "bank_account": 0.0,
        "student_loan": 0.0,
        "money_transfer": 0.0
       },
       "probability_of_expected": 0.9999,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "credit_reporting",
       "correct": true,
       "probabilities": {
        "credit_reporting": 0.5843,
        "debt_collection": 0.3387,
        "credit_card": 0.0223,
        "mortgage": 0.0209,
        "money_transfer": 0.0124,
        "bank_account": 0.0107
       },
       "probability_of_expected": 0.5843,
       "proper_score": 0.8556
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2860.1,
    "Jev": 309.1,
    "JevFish": 1566.2,
    "NanoJev": 699.5,
    "OpenJev": 352.1,
    "System One Scorer": 2834.9
   }
  },
  {
   "id": "workflow-control:cfpb-c0bed675e232",
   "state": {
    "complaint": "I bought my home with a government backed FHA loan in XX/XX/XXXX. I am now facing foreclosure because of a misunderstanding regarding the mortgage payment. I have been told that there are no options for help besides full reinstatement or an expensive repayment plan. I feel that Carrington Mortgage has obstructed access to government programs that should be made available to me since I have a government backed loan. I have never been considered for a loan modification or a partial claim and was not even aware that there was a problem until two months ago. When I bought the home I lived here with my boyfriend and XXXX XXXX. My boyfriend covered the mortgage payment and I took care of all of the bills and groceries. In XX/XX/XXXX my boyfriend separated from me and moved out. Since we share a XXXX  we made an agreement that he would continue …"
   },
   "state_tokens": 179,
   "tier": "common",
   "primary": "product_queue",
   "questions": [
    {
     "id": "product_queue",
     "type": "choice",
     "instructions": "Read the `complaint` narrative. Choose the single financial product line the consumer is complaining about, i.e. the specialist queue that should own this ticket. Personal details are redacted as XXXX.",
     "options": [
      [
       "bank_account",
       "Checking or savings account, or bank account services (deposits, overdrafts, holds, closures, ATM/debit issues)"
      ],
      [
       "credit_card",
       "Credit card or prepaid card (charges, rewards, card fees, card fraud, card billing disputes)"
      ],
      [
       "credit_reporting",
       "Credit reporting agencies, credit repair, or other consumer reports (incorrect report data, disputes with bureaus, identity-theft entries)"
      ],
      [
       "debt_collection",
       "A third-party or first-party debt collector's conduct or a debt the consumer says is not theirs"
      ],
      [
       "money_transfer",
       "Money transfer, virtual currency, or money services (wires, P2P apps, remittances, crypto, check cashing)"
      ],
      [
       "mortgage",
       "Home mortgage loans (origination, servicing, escrow, modification, foreclosure)"
      ],
      [
       "student_loan",
       "Federal or private student loans (servicing, repayment plans, forgiveness, collections on student debt)"
      ]
     ],
     "graded": true,
     "expected": "mortgage",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "mortgage",
       "correct": true,
       "probabilities": {
        "mortgage": 0.9993,
        "credit_reporting": 0.0002,
        "bank_account": 0.0001,
        "credit_card": 0.0001,
        "debt_collection": 0.0001,
        "money_transfer": 0.0001
       },
       "probability_of_expected": 0.9993,
       "proper_score": 1.0
      },
      "Jev": {
       "answered": true,
       "pick": "mortgage",
       "correct": true,
       "probabilities": {
        "mortgage": 1.0,
        "student_loan": 0.0,
        "credit_card": 0.0,
        "debt_collection": 0.0,
        "credit_reporting": 0.0,
        "bank_account": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "mortgage",
       "correct": true,
       "probabilities": {
        "mortgage": 0.9975,
        "student_loan": 0.0008,
        "money_transfer": 0.0004,
        "bank_account": 0.0004,
        "debt_collection": 0.0004,
        "credit_card": 0.0003
       },
       "probability_of_expected": 0.9975,
       "proper_score": 1.0
      },
      "NanoJev": {
       "answered": true,
       "pick": "mortgage",
       "correct": true,
       "probabilities": {
        "mortgage": 0.6936,
        "debt_collection": 0.1264,
        "credit_reporting": 0.0613,
        "student_loan": 0.0463,
        "money_transfer": 0.0407,
        "credit_card": 0.0198
       },
       "probability_of_expected": 0.6936,
       "proper_score": 0.941
      },
      "OpenJev": {
       "answered": true,
       "pick": "mortgage",
       "correct": true,
       "probabilities": {
        "mortgage": 0.9999,
        "bank_account": 0.0,
        "credit_card": 0.0,
        "debt_collection": 0.0,
        "student_loan": 0.0,
        "money_transfer": 0.0
       },
       "probability_of_expected": 0.9999,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "mortgage",
       "correct": true,
       "probabilities": {
        "mortgage": 0.9096,
        "debt_collection": 0.0501,
        "student_loan": 0.0134,
        "money_transfer": 0.008,
        "bank_account": 0.0074,
        "credit_reporting": 0.0063
       },
       "probability_of_expected": 0.9096,
       "proper_score": 0.9945
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2352.7,
    "Jev": 162.5,
    "JevFish": 1520.7,
    "NanoJev": 788.2,
    "OpenJev": 296.3,
    "System One Scorer": 2933.1
   }
  },
  {
   "id": "workflow-control:cfpb-f0289149e030",
   "state": {
    "complaint": "Applied for SAVE XX/XX/2023. Advised by Student Aid department that application would take XXXX days. Application approved XX/XX/2023. Received mail from MOHELA confirming application had processed successfully and application had entered SAVE repayment. Setup auto debit online - confirmed due date with first payment in XXXX. \nReviewed website XX/XX/2023 - loans now placed in Forbearance. Call center first told me it was due to natural disaster is state for which I was not affected - XXXX minutes later, phone representative says it is a processing forbearance and they will charge me for the interest. \nApplication had already been approved, auto debit had been approved, mail received confirming due date in XXXX in early XXXX. MOHELA  is unable to confirm info due to long wait lines but I did not consent to this forbearance and it is …"
   },
   "state_tokens": 195,
   "tier": "common",
   "primary": "product_queue",
   "questions": [
    {
     "id": "product_queue",
     "type": "choice",
     "instructions": "Read the `complaint` narrative. Choose the single financial product line the consumer is complaining about, i.e. the specialist queue that should own this ticket. Personal details are redacted as XXXX.",
     "options": [
      [
       "bank_account",
       "Checking or savings account, or bank account services (deposits, overdrafts, holds, closures, ATM/debit issues)"
      ],
      [
       "credit_card",
       "Credit card or prepaid card (charges, rewards, card fees, card fraud, card billing disputes)"
      ],
      [
       "credit_reporting",
       "Credit reporting agencies, credit repair, or other consumer reports (incorrect report data, disputes with bureaus, identity-theft entries)"
      ],
      [
       "debt_collection",
       "A third-party or first-party debt collector's conduct or a debt the consumer says is not theirs"
      ],
      [
       "money_transfer",
       "Money transfer, virtual currency, or money services (wires, P2P apps, remittances, crypto, check cashing)"
      ],
      [
       "mortgage",
       "Home mortgage loans (origination, servicing, escrow, modification, foreclosure)"
      ],
      [
       "student_loan",
       "Federal or private student loans (servicing, repayment plans, forgiveness, collections on student debt)"
      ]
     ],
     "graded": true,
     "expected": "student_loan",
     "responses": {
      "decider-2b": {
       "answered": true,
       "pick": "student_loan",
       "correct": true,
       "probabilities": {
        "student_loan": 0.9987,
        "mortgage": 0.0009,
        "bank_account": 0.0001,
        "credit_card": 0.0001,
        "credit_reporting": 0.0001,
        "debt_collection": 0.0001
       },
       "probability_of_expected": 0.9987,
       "proper_score": 1.0
      },
      "Jev": {
       "answered": true,
       "pick": "student_loan",
       "correct": true,
       "probabilities": {
        "student_loan": 1.0,
        "bank_account": 0.0,
        "debt_collection": 0.0,
        "money_transfer": 0.0,
        "credit_card": 0.0,
        "credit_reporting": 0.0
       },
       "probability_of_expected": 1.0,
       "proper_score": 1.0
      },
      "JevFish": {
       "answered": true,
       "pick": "student_loan",
       "correct": true,
       "probabilities": {
        "student_loan": 0.988,
        "mortgage": 0.0062,
        "debt_collection": 0.0024,
        "bank_account": 0.0016,
        "credit_reporting": 0.0008,
        "credit_card": 0.0007
       },
       "probability_of_expected": 0.988,
       "proper_score": 0.9999
      },
      "NanoJev": {
       "answered": true,
       "pick": "debt_collection",
       "correct": false,
       "probabilities": {
        "debt_collection": 0.5212,
        "student_loan": 0.2395,
        "bank_account": 0.0949,
        "money_transfer": 0.058,
        "credit_reporting": 0.0438,
        "credit_card": 0.0269
       },
       "probability_of_expected": 0.2395,
       "proper_score": 0.5674
      },
      "OpenJev": {
       "answered": true,
       "pick": "student_loan",
       "correct": true,
       "probabilities": {
        "student_loan": 0.9998,
        "bank_account": 0.0001,
        "credit_card": 0.0001,
        "debt_collection": 0.0,
        "mortgage": 0.0,
        "money_transfer": 0.0
       },
       "probability_of_expected": 0.9998,
       "proper_score": 1.0
      },
      "System One Scorer": {
       "answered": true,
       "pick": "student_loan",
       "correct": true,
       "probabilities": {
        "student_loan": 0.8442,
        "mortgage": 0.0817,
        "debt_collection": 0.0432,
        "bank_account": 0.0125,
        "money_transfer": 0.0072,
        "credit_card": 0.0056
       },
       "probability_of_expected": 0.8442,
       "proper_score": 0.9835
      }
     }
    }
   ],
   "latency_ms": {
    "decider-2b": 2370.3,
    "Jev": 178.9,
    "JevFish": 1516.3,
    "NanoJev": 718.8,
    "OpenJev": 306.5,
    "System One Scorer": 2740.3
   }
  }
 ]
}