{ "schemaVersion": "blobfish.typed-decision-bench-tasks.v1", "version": "0.3.0", "observedAt": "2026-09-20T20:01:35Z", "systems": [ "Jev", "JevFish", "NanoJev", "OpenJev", "System One Scorer", "decider-2b" ], "tasks": [ { "task": "abt-buy-product-match", "family": "retrieval-and-knowledge", "use_case": "entity-alignment", "vertical": "e-commerce", "dataset": "matchbench/Abt-Buy", "licence": "CC (variant unconfirmed on the source page) — FLAGGED", "provenance": "expert", "primitive": "score", "items": 200, "scores": { "decider-2b": { "decisionScore": 82.7, "accuracy": 0.905, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 94.91, "accuracy": 0.93, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 87.1, "accuracy": 0.895, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 74.54, "accuracy": 0.665, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 92.91, "accuracy": 0.92, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 85.37, "accuracy": 0.865, "answered": 200, "items": 200 } } }, { "task": "arxiv-archive-subcategory", "family": "data-and-operations", "use_case": "hierarchical-classification", "vertical": "science-publishing", "dataset": "gfissore/arxiv-abstracts-2021", "licence": "CC0-1.0", "provenance": "outcome", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 70.09, "accuracy": 0.78, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 76.85, "accuracy": 0.885, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 70.72, "accuracy": 0.81, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 53.67, "accuracy": 0.295, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 65.41, "accuracy": 0.865, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 70.57, "accuracy": 0.815, "answered": 200, "items": 200 } } }, { "task": "bfcl-tool-dispatch", "family": "real-time-and-agents", "use_case": "agent-routing-and-skill-selection", "vertical": "software-engineering", "dataset": "minpeter/bfcl-v1-non-live-ast-parsed", "licence": "Apache-2.0", "provenance": "expert", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 92.13, "accuracy": 0.89, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 97.56, "accuracy": 0.97, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 93.01, "accuracy": 0.895, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 61.03, "accuracy": 0.4541, "answered": 196, "items": 200 }, "OpenJev": { "decisionScore": 96.45, "accuracy": 0.96, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 83.18, "accuracy": 0.79, "answered": 200, "items": 200 } } }, { "task": "casehold-cited-holding", "family": "retrieval-and-knowledge", "use_case": "citation-verification", "vertical": "legal", "dataset": "coastalcph/lex_glue", "licence": "CC-BY-4.0", "provenance": "outcome", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 79.98, "accuracy": 0.705, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 87.03, "accuracy": 0.795, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 80.0, "accuracy": 0.745, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 61.18, "accuracy": 0.3384, "answered": 198, "items": 200 }, "OpenJev": { "decisionScore": 74.69, "accuracy": 0.665, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 69.53, "accuracy": 0.515, "answered": 200, "items": 200 } } }, { "task": "cfpb-complaint-product-queue", "family": "workflow-control", "use_case": "support-inbox-triage", "vertical": "finance", "dataset": "liri-uzh/cfpb-complaints-mini", "licence": "CC0-1.0 (CFPB public domain)", "provenance": "outcome", "primitive": "choice", "items": 199, "scores": { "decider-2b": { "decisionScore": 79.71, "accuracy": 0.7286, "answered": 199, "items": 199 }, "Jev": { "decisionScore": 83.45, "accuracy": 0.794, "answered": 199, "items": 199 }, "JevFish": { "decisionScore": 78.76, "accuracy": 0.7136, "answered": 199, "items": 199 }, "NanoJev": { "decisionScore": 63.41, "accuracy": 0.4372, "answered": 199, "items": 199 }, "OpenJev": { "decisionScore": 78.94, "accuracy": 0.7688, "answered": 199, "items": 199 }, "System One Scorer": { "decisionScore": 82.62, "accuracy": 0.7789, "answered": 199, "items": 199 } } }, { "task": "cuad-checklist", "family": "safety-and-quality", "use_case": "compliance-verification", "vertical": "legal", "dataset": "nguha/legalbench", "licence": "CC-BY-4.0", "provenance": "expert", "primitive": "noul", "items": 204, "scores": { "decider-2b": { "decisionScore": 95.1, "accuracy": 0.951, "answered": 204, "items": 204 }, "Jev": { "decisionScore": 91.61, "accuracy": 0.8922, "answered": 204, "items": 204 }, "JevFish": { "decisionScore": 93.6, "accuracy": 0.9314, "answered": 204, "items": 204 }, "NanoJev": { "decisionScore": 72.65, "accuracy": 0.4951, "answered": 204, "items": 204 }, "OpenJev": { "decisionScore": 91.22, "accuracy": 0.8922, "answered": 204, "items": 204 }, "System One Scorer": { "decisionScore": 92.1, "accuracy": 0.9265, "answered": 204, "items": 204 } } }, { "task": "dbpedia-ontology-hierarchy", "family": "data-and-operations", "use_case": "hierarchical-classification", "vertical": "media-and-reference", "dataset": "DeveloperOats/DBPedia_Classes", "licence": "CC0-1.0", "provenance": "outcome", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 89.04, "accuracy": 0.89, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 92.86, "accuracy": 0.94, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 89.58, "accuracy": 0.9, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 60.64, "accuracy": 0.155, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 89.31, "accuracy": 0.925, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 83.82, "accuracy": 0.87, "answered": 200, "items": 200 } } }, { "task": "esci-product-rerank", "family": "retrieval-and-knowledge", "use_case": "semantic-reranking", "vertical": "e-commerce", "dataset": "tasksource/esci", "licence": "Apache-2.0", "provenance": "crowd", "primitive": "noul", "items": 490, "scores": { "decider-2b": { "decisionScore": 74.15, "accuracy": 0.6204, "answered": 490, "items": 490 }, "Jev": { "decisionScore": 74.99, "accuracy": 0.7306, "answered": 490, "items": 490 }, "JevFish": { "decisionScore": 74.41, "accuracy": 0.6306, "answered": 490, "items": 490 }, "NanoJev": { "decisionScore": 71.08, "accuracy": 0.593, "answered": 489, "items": 490 }, "OpenJev": { "decisionScore": 71.82, "accuracy": 0.7224, "answered": 490, "items": 490 }, "System One Scorer": { "decisionScore": 75.51, "accuracy": 0.6163, "answered": 490, "items": 490 } } }, { "task": "finer-xbrl-fact-tag", "family": "data-and-operations", "use_case": "structured-extraction", "vertical": "finance", "dataset": "nlpaueb/finer-139", "licence": "CC-BY-SA-4.0", "provenance": "expert", "primitive": "choice", "items": 181, "scores": { "decider-2b": { "decisionScore": 79.81, "accuracy": 0.7293, "answered": 181, "items": 181 }, "Jev": { "decisionScore": 93.3, "accuracy": 0.9227, "answered": 181, "items": 181 }, "JevFish": { "decisionScore": 82.71, "accuracy": 0.7735, "answered": 181, "items": 181 }, "NanoJev": { "decisionScore": 53.03, "accuracy": 0.1381, "answered": 181, "items": 181 }, "OpenJev": { "decisionScore": 88.96, "accuracy": 0.8508, "answered": 181, "items": 181 }, "System One Scorer": { "decisionScore": 79.57, "accuracy": 0.8011, "answered": 181, "items": 181 } } }, { "task": "hotpot-context-filter", "family": "retrieval-and-knowledge", "use_case": "rag-passage-filtering", "vertical": "knowledge-work", "dataset": "hotpotqa/hotpot_qa", "licence": "CC-BY-SA-4.0", "provenance": "crowd", "primitive": "noul", "items": 200, "scores": { "decider-2b": { "decisionScore": 86.67, "accuracy": 0.84, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 96.3, "accuracy": 0.945, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 90.67, "accuracy": 0.825, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 0.0, "accuracy": null, "answered": 0, "items": 200 }, "OpenJev": { "decisionScore": 85.71, "accuracy": 0.93, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 75.4, "accuracy": 0.59, "answered": 200, "items": 200 } } }, { "task": "humanevalpack-spec-match", "family": "safety-and-quality", "use_case": "semantic-linting", "vertical": "software-engineering", "dataset": "bigcode/humanevalpack", "licence": "MIT", "provenance": "verifier", "primitive": "noul", "items": 164, "scores": { "decider-2b": { "decisionScore": 71.39, "accuracy": 0.6524, "answered": 164, "items": 164 }, "Jev": { "decisionScore": 82.84, "accuracy": 0.8902, "answered": 164, "items": 164 }, "JevFish": { "decisionScore": 71.62, "accuracy": 0.628, "answered": 164, "items": 164 }, "NanoJev": { "decisionScore": 68.07, "accuracy": 0.5031, "answered": 163, "items": 164 }, "OpenJev": { "decisionScore": 80.53, "accuracy": 0.8841, "answered": 164, "items": 164 }, "System One Scorer": { "decisionScore": 73.16, "accuracy": 0.6585, "answered": 164, "items": 164 } } }, { "task": "hwu64-tool-select", "family": "workflow-control", "use_case": "typed-tool-dispatch", "vertical": "consumer-assistant", "dataset": "deutsche-telekom/NLU-Evaluation-Data-en-de", "licence": "CC-BY-4.0", "provenance": "crowd", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 91.17, "accuracy": 0.86, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 93.66, "accuracy": 0.88, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 91.85, "accuracy": 0.88, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 67.8, "accuracy": 0.36, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 91.45, "accuracy": 0.87, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 83.37, "accuracy": 0.805, "answered": 200, "items": 200 } } }, { "task": "k8s-issue-kind-routing", "family": "real-time-and-agents", "use_case": "agent-routing-and-skill-selection", "vertical": "devops", "dataset": "open-index/open-github-issues", "licence": "ODC-BY-1.0", "provenance": "outcome", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 75.36, "accuracy": 0.66, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 77.46, "accuracy": 0.725, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 75.54, "accuracy": 0.645, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 61.47, "accuracy": 0.34, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 73.84, "accuracy": 0.71, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 73.46, "accuracy": 0.61, "answered": 200, "items": 200 } } }, { "task": "ledgar-clause-review-routing", "family": "workflow-control", "use_case": "intent-and-model-routing", "vertical": "legal", "dataset": "coastalcph/lex_glue", "licence": "CC-BY-4.0", "provenance": "outcome", "primitive": "choice", "items": 199, "scores": { "decider-2b": { "decisionScore": 37.3, "accuracy": 0.2513, "answered": 199, "items": 199 }, "Jev": { "decisionScore": 34.55, "accuracy": 0.2864, "answered": 199, "items": 199 }, "JevFish": { "decisionScore": 41.45, "accuracy": 0.2663, "answered": 199, "items": 199 }, "NanoJev": { "decisionScore": 50.8, "accuracy": 0.1407, "answered": 199, "items": 199 }, "OpenJev": { "decisionScore": 32.75, "accuracy": 0.2814, "answered": 199, "items": 199 }, "System One Scorer": { "decisionScore": 48.89, "accuracy": 0.201, "answered": 199, "items": 199 } } }, { "task": "moderation-hate-severity", "family": "safety-and-quality", "use_case": "llm-guardrails", "vertical": "trust-and-safety", "dataset": "mmathys/openai-moderation-api-evaluation", "licence": "MIT", "provenance": "expert", "primitive": "score", "items": 200, "scores": { "decider-2b": { "decisionScore": 89.64, "accuracy": 0.78, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 93.38, "accuracy": 0.85, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 90.08, "accuracy": 0.79, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 75.03, "accuracy": 0.35, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 90.31, "accuracy": 0.775, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 86.32, "accuracy": 0.74, "answered": 200, "items": 200 } } }, { "task": "prompt-injection", "family": "safety-and-quality", "use_case": "llm-guardrails", "vertical": "security", "dataset": "deepset/prompt-injections", "licence": "Apache-2.0", "provenance": "crowd", "primitive": "noul", "items": 194, "scores": { "decider-2b": { "decisionScore": 76.3, "accuracy": 0.701, "answered": 194, "items": 194 }, "Jev": { "decisionScore": 90.54, "accuracy": 0.8763, "answered": 194, "items": 194 }, "JevFish": { "decisionScore": 79.12, "accuracy": 0.7165, "answered": 194, "items": 194 }, "NanoJev": { "decisionScore": 77.92, "accuracy": 0.6443, "answered": 194, "items": 194 }, "OpenJev": { "decisionScore": 85.38, "accuracy": 0.8351, "answered": 194, "items": 194 }, "System One Scorer": { "decisionScore": 82.43, "accuracy": 0.7577, "answered": 194, "items": 194 } } }, { "task": "swe-rebench-run-resolved", "family": "real-time-and-agents", "use_case": "run-judging", "vertical": "software-engineering", "dataset": "nebius/SWE-rebench-openhands-trajectories", "licence": "CC-BY-4.0", "provenance": "verifier", "primitive": "noul", "items": 200, "scores": { "decider-2b": { "decisionScore": 69.39, "accuracy": 0.52, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 74.88, "accuracy": 0.6, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 75.3, "accuracy": 0.565, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 0.0, "accuracy": null, "answered": 0, "items": 200 }, "OpenJev": { "decisionScore": 59.23, "accuracy": 0.555, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 73.45, "accuracy": 0.5, "answered": 200, "items": 200 } } }, { "task": "swe-smith-run-resolved", "family": "real-time-and-agents", "use_case": "run-judging", "vertical": "software-engineering", "dataset": "SWE-bench/SWE-smith-trajectories", "licence": "MIT", "provenance": "verifier", "primitive": "noul", "items": 200, "scores": { "decider-2b": { "decisionScore": 69.76, "accuracy": 0.46, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 60.2, "accuracy": 0.505, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 63.74, "accuracy": 0.535, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 0.33, "accuracy": 0.0, "answered": 1, "items": 200 }, "OpenJev": { "decisionScore": 51.14, "accuracy": 0.495, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 71.84, "accuracy": 0.5, "answered": 200, "items": 200 } } }, { "task": "tau2-airline-run-success", "family": "real-time-and-agents", "use_case": "run-judging", "vertical": "customer-operations", "dataset": "snorkelai/Tau2-Bench-Verified-Airline-With-Code-Agents", "licence": "Apache-2.0", "provenance": "verifier", "primitive": "noul", "items": 200, "scores": { "decider-2b": { "decisionScore": 67.32, "accuracy": 0.47, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 68.02, "accuracy": 0.52, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 64.89, "accuracy": 0.515, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 75.47, "accuracy": 0.5, "answered": 198, "items": 200 }, "OpenJev": { "decisionScore": 55.26, "accuracy": 0.475, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 71.85, "accuracy": 0.485, "answered": 200, "items": 200 } } }, { "task": "tau2-airline-transcript-success", "family": "real-time-and-agents", "use_case": "run-judging", "vertical": "customer-operations", "dataset": "snorkelai/Tau2-Bench-Verified-Airline-With-Code-Agents", "licence": "Apache-2.0", "provenance": "verifier", "primitive": "noul", "items": 199, "scores": { "decider-2b": { "decisionScore": 63.88, "accuracy": 0.3166, "answered": 199, "items": 199 }, "Jev": { "decisionScore": 76.74, "accuracy": 0.608, "answered": 199, "items": 199 }, "JevFish": { "decisionScore": 65.67, "accuracy": 0.3668, "answered": 199, "items": 199 }, "NanoJev": { "decisionScore": 1.36, "accuracy": 0.6667, "answered": 3, "items": 199 }, "OpenJev": { "decisionScore": 63.68, "accuracy": 0.4874, "answered": 199, "items": 199 }, "System One Scorer": { "decisionScore": 74.83, "accuracy": 0.5276, "answered": 199, "items": 199 } } }, { "task": "trial-outcomes", "family": "data-and-operations", "use_case": "feature-extraction-for-ml", "vertical": "saas-growth", "dataset": "synthetic:saas-trial-v1", "licence": "CC-BY-4.0 (generated here)", "provenance": "oracle", "primitive": "noul", "items": 400, "scores": { "decider-2b": { "decisionScore": 77.85, "accuracy": 0.695, "answered": 400, "items": 400 }, "Jev": { "decisionScore": 78.75, "accuracy": 0.6775, "answered": 400, "items": 400 }, "JevFish": { "decisionScore": 75.35, "accuracy": 0.695, "answered": 400, "items": 400 }, "NanoJev": { "decisionScore": 76.4, "accuracy": 0.695, "answered": 400, "items": 400 }, "OpenJev": { "decisionScore": 69.41, "accuracy": 0.6925, "answered": 400, "items": 400 }, "System One Scorer": { "decisionScore": 77.74, "accuracy": 0.695, "answered": 400, "items": 400 } } }, { "task": "trialgpt-criterion-support", "family": "retrieval-and-knowledge", "use_case": "citation-verification", "vertical": "healthcare", "dataset": "ncbi/TrialGPT-Criterion-Annotations", "licence": "Public domain (NCBI)", "provenance": "expert", "primitive": "choice", "items": 159, "scores": { "decider-2b": { "decisionScore": 73.31, "accuracy": 0.5786, "answered": 159, "items": 159 }, "Jev": { "decisionScore": 73.79, "accuracy": 0.6415, "answered": 159, "items": 159 }, "JevFish": { "decisionScore": 73.51, "accuracy": 0.6164, "answered": 159, "items": 159 }, "NanoJev": { "decisionScore": 58.71, "accuracy": 0.1258, "answered": 159, "items": 159 }, "OpenJev": { "decisionScore": 60.57, "accuracy": 0.5346, "answered": 159, "items": 159 }, "System One Scorer": { "decisionScore": 70.57, "accuracy": 0.5346, "answered": 159, "items": 159 } } }, { "task": "unfair-tos-clause-type", "family": "safety-and-quality", "use_case": "compliance-verification", "vertical": "consumer-legal", "dataset": "nguha/legalbench", "licence": "CC-BY-4.0", "provenance": "expert", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 83.53, "accuracy": 0.8, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 85.91, "accuracy": 0.82, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 87.26, "accuracy": 0.835, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 60.83, "accuracy": 0.255, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 83.92, "accuracy": 0.805, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 79.07, "accuracy": 0.78, "answered": 200, "items": 200 } } }, { "task": "unfair-tos-publish-gate", "family": "workflow-control", "use_case": "confidence-gated-actions", "vertical": "consumer-legal", "dataset": "coastalcph/lex_glue", "licence": "CC-BY-4.0", "provenance": "expert", "primitive": "noul", "items": 198, "scores": { "decider-2b": { "decisionScore": 83.27, "accuracy": 0.7576, "answered": 198, "items": 198 }, "Jev": { "decisionScore": 87.1, "accuracy": 0.8434, "answered": 198, "items": 198 }, "JevFish": { "decisionScore": 83.13, "accuracy": 0.7071, "answered": 198, "items": 198 }, "NanoJev": { "decisionScore": 75.97, "accuracy": 0.6263, "answered": 198, "items": 198 }, "OpenJev": { "decisionScore": 83.9, "accuracy": 0.8182, "answered": 198, "items": 198 }, "System One Scorer": { "decisionScore": 83.92, "accuracy": 0.8232, "answered": 198, "items": 198 } } }, { "task": "vitaminc-claim-consistency", "family": "safety-and-quality", "use_case": "self-consistency-checks", "vertical": "media", "dataset": "tals/vitaminc", "licence": "CC-BY-SA-3.0", "provenance": "crowd", "primitive": "choice", "items": 200, "scores": { "decider-2b": { "decisionScore": 82.42, "accuracy": 0.715, "answered": 200, "items": 200 }, "Jev": { "decisionScore": 88.08, "accuracy": 0.83, "answered": 200, "items": 200 }, "JevFish": { "decisionScore": 83.26, "accuracy": 0.73, "answered": 200, "items": 200 }, "NanoJev": { "decisionScore": 60.8, "accuracy": 0.505, "answered": 200, "items": 200 }, "OpenJev": { "decisionScore": 82.77, "accuracy": 0.77, "answered": 200, "items": 200 }, "System One Scorer": { "decisionScore": 82.32, "accuracy": 0.76, "answered": 200, "items": 200 } } } ] }