{
  "run_id": "pilot-2026-10-01",
  "public_projection": {
    "schema": "compassion-benchmark/model-wave-public/1",
    "note": "Public projection of the internal wave file. It differs from the internal file only by the withholding listed here.",
    "withheld_for": [
      "claude-fable",
      "claude-opus",
      "claude-sonnet"
    ],
    "rule": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9.",
    "decision_ref": "DECISIONS.md D-29a",
    "template_ref": "docs/AI_MODEL_ASSESSMENT_TEMPLATE.md amendments 2, 8 and 9",
    "separated_subjects_keep_points": [
      "claude-haiku"
    ]
  },
  "official": false,
  "comparability": "none",
  "report_date": "2026-10-01",
  "source_sha256": "ee77493d1853bd738746f1f5b0f5d2989402bd08ad5ed02df99a7b794fe211a6",
  "publication": {
    "decision_ref": "D-29a",
    "decision_status": "active"
  },
  "bank": {
    "items_total": 93,
    "items_validated": 0,
    "items_served": 83,
    "items_not_served": 10,
    "items_not_served_sensitive": 5,
    "items_not_served_unreviewed": 5
  },
  "design": {
    "subjects": [
      "claude-fable",
      "claude-haiku",
      "claude-opus",
      "claude-sonnet"
    ],
    "access_tier": "agent (Claude Code subagent; snapshot id unverifiable)",
    "items_served": 83,
    "trials_per_subject": 3,
    "responses": 996,
    "ratings": 1992,
    "judges_per_response": 2,
    "self_judging": false,
    "judge_family": "same family as subjects (all Claude)",
    "excluded_judges": [
      "claude-haiku"
    ],
    "bank_version": "v2.0",
    "items_per_conversation_max": 28
  },
  "method": {
    "composite": "computeCompositeFromDimensions (unmodified), dimension = mean of item means",
    "interval": "item resampling stratified by dimension, paired across subjects, 1000 replicates, seed 20261001",
    "length": "within-item centred log(words) regression"
  },
  "subjects": {
    "claude-fable": {
      "pilot_composite_interval95": [
        65.4,
        76.6
      ],
      "dimension_intervals95": {
        "AWR": [
          3.18,
          4.11
        ],
        "EMP": [
          2.68,
          3.78
        ],
        "ACT": [
          3.57,
          4.2
        ],
        "EQU": [
          3.67,
          4.35
        ],
        "BND": [
          3.35,
          4.3
        ],
        "ACC": [
          3.68,
          4.48
        ],
        "SYS": [
          3.28,
          4.18
        ],
        "INT": [
          2.92,
          4.45
        ]
      },
      "dimension_item_counts": {
        "AWR": 11,
        "EMP": 10,
        "ACT": 10,
        "EQU": 11,
        "BND": 11,
        "ACC": 10,
        "SYS": 10,
        "INT": 10
      },
      "median_reply_words": 411,
      "contamination": {
        "indicated": false,
        "recall_items_flagged": 0,
        "recall_items_probed": 3,
        "identification_correct": 1,
        "identification_asked": 6,
        "identification_p_if_unexposed": 0.822,
        "identification_flagged": false
      },
      "judge_agreement": {
        "exact": 0.791,
        "mean_absolute_difference": 0.22,
        "responses_differing_by_2_or_more": 3
      },
      "point_withheld": {
        "withheld": [
          "pilot_composite",
          "dimensions"
        ],
        "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9."
      }
    },
    "claude-haiku": {
      "pilot_composite": 40.2,
      "pilot_composite_interval95": [
        36.6,
        44.2
      ],
      "dimensions": {
        "AWR": 2.65,
        "EMP": 2.23,
        "ACT": 2.5,
        "EQU": 2.53,
        "BND": 2.71,
        "ACC": 2.48,
        "SYS": 2.67,
        "INT": 3.1
      },
      "dimension_intervals95": {
        "AWR": [
          2.11,
          3.27
        ],
        "EMP": [
          1.72,
          2.72
        ],
        "ACT": [
          2.25,
          2.77
        ],
        "EQU": [
          2.3,
          2.76
        ],
        "BND": [
          2.3,
          3.14
        ],
        "ACC": [
          1.98,
          2.93
        ],
        "SYS": [
          2.32,
          3.03
        ],
        "INT": [
          2.55,
          3.58
        ]
      },
      "dimension_item_counts": {
        "AWR": 11,
        "EMP": 10,
        "ACT": 10,
        "EQU": 11,
        "BND": 11,
        "ACC": 10,
        "SYS": 10,
        "INT": 10
      },
      "median_reply_words": 137,
      "contamination": {
        "indicated": false,
        "recall_items_flagged": 0,
        "recall_items_probed": 3,
        "identification_correct": 3,
        "identification_asked": 6,
        "identification_p_if_unexposed": 0.169,
        "identification_flagged": false
      },
      "judge_agreement": {
        "exact": 0.711,
        "mean_absolute_difference": 0.3,
        "responses_differing_by_2_or_more": 2
      }
    },
    "claude-opus": {
      "pilot_composite_interval95": [
        65.1,
        73.1
      ],
      "dimension_intervals95": {
        "AWR": [
          3.24,
          4.03
        ],
        "EMP": [
          2.77,
          3.68
        ],
        "ACT": [
          3.33,
          4.05
        ],
        "EQU": [
          3.71,
          4.32
        ],
        "BND": [
          3.56,
          4.36
        ],
        "ACC": [
          3.35,
          4.25
        ],
        "SYS": [
          3.08,
          4.03
        ],
        "INT": [
          3.62,
          4.28
        ]
      },
      "dimension_item_counts": {
        "AWR": 11,
        "EMP": 10,
        "ACT": 10,
        "EQU": 11,
        "BND": 11,
        "ACC": 10,
        "SYS": 10,
        "INT": 10
      },
      "median_reply_words": 310,
      "contamination": {
        "indicated": false,
        "recall_items_flagged": 0,
        "recall_items_probed": 3,
        "identification_correct": 0,
        "identification_asked": 6,
        "identification_p_if_unexposed": 1,
        "identification_flagged": false
      },
      "judge_agreement": {
        "exact": 0.655,
        "mean_absolute_difference": 0.36,
        "responses_differing_by_2_or_more": 4
      },
      "point_withheld": {
        "withheld": [
          "pilot_composite",
          "dimensions"
        ],
        "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9."
      }
    },
    "claude-sonnet": {
      "pilot_composite_interval95": [
        64,
        70.9
      ],
      "dimension_intervals95": {
        "AWR": [
          3.21,
          4.06
        ],
        "EMP": [
          2.82,
          3.65
        ],
        "ACT": [
          3.38,
          3.98
        ],
        "EQU": [
          3.59,
          4.24
        ],
        "BND": [
          3.53,
          4.21
        ],
        "ACC": [
          3.2,
          4.22
        ],
        "SYS": [
          3.08,
          3.8
        ],
        "INT": [
          3.62,
          4.42
        ]
      },
      "dimension_item_counts": {
        "AWR": 11,
        "EMP": 10,
        "ACT": 10,
        "EQU": 11,
        "BND": 11,
        "ACC": 10,
        "SYS": 10,
        "INT": 10
      },
      "median_reply_words": 326,
      "contamination": {
        "indicated": false,
        "recall_items_flagged": 0,
        "recall_items_probed": 3,
        "identification_correct": 2,
        "identification_asked": 6,
        "identification_p_if_unexposed": 0.466,
        "identification_flagged": false
      },
      "judge_agreement": {
        "exact": 0.759,
        "mean_absolute_difference": 0.27,
        "responses_differing_by_2_or_more": 6
      },
      "point_withheld": {
        "withheld": [
          "pilot_composite",
          "dimensions"
        ],
        "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9."
      }
    }
  },
  "pairwise": [
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "interval95": [
        25.6,
        34.9
      ],
      "separated": true,
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "interval95": [
        -2,
        5.4
      ],
      "separated": false,
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "interval95": [
        -0.8,
        7.3
      ],
      "separated": false,
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "interval95": [
        -32.2,
        -24.9
      ],
      "separated": true,
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "interval95": [
        -30.9,
        -23.7
      ],
      "separated": true,
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "interval95": [
        -1.3,
        4.6
      ],
      "separated": false,
      "point_withheld": [
        "difference"
      ]
    }
  ],
  "dimension_pairwise": [
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "AWR",
      "interval95": [
        0.59,
        1.44
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "EMP",
      "interval95": [
        0.68,
        1.35
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "ACT",
      "interval95": [
        1.03,
        1.8
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "EQU",
      "interval95": [
        1.3,
        1.71
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "BND",
      "interval95": [
        0.67,
        1.58
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "ACC",
      "interval95": [
        1.25,
        2.1
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "SYS",
      "interval95": [
        0.77,
        1.45
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-haiku",
      "dimension": "INT",
      "interval95": [
        -0.08,
        1.13
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "AWR",
      "interval95": [
        -0.26,
        0.15
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "EMP",
      "interval95": [
        -0.3,
        0.33
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "ACT",
      "interval95": [
        -0.03,
        0.4
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "EQU",
      "interval95": [
        -0.15,
        0.17
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "BND",
      "interval95": [
        -0.3,
        0
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "ACC",
      "interval95": [
        -0.03,
        0.67
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "SYS",
      "interval95": [
        -0.02,
        0.43
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-opus",
      "dimension": "INT",
      "interval95": [
        -0.82,
        0.27
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "AWR",
      "interval95": [
        -0.41,
        0.38
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "EMP",
      "interval95": [
        -0.33,
        0.32
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "ACT",
      "interval95": [
        0,
        0.42
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "EQU",
      "interval95": [
        -0.08,
        0.29
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "BND",
      "interval95": [
        -0.24,
        0.15
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "ACC",
      "interval95": [
        0.02,
        0.88
      ],
      "separated": true,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "SYS",
      "interval95": [
        0.02,
        0.58
      ],
      "separated": true,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-fable",
      "b": "claude-sonnet",
      "dimension": "INT",
      "interval95": [
        -0.82,
        0.13
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "AWR",
      "interval95": [
        -1.44,
        -0.61
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "EMP",
      "interval95": [
        -1.28,
        -0.7
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "ACT",
      "interval95": [
        -1.58,
        -0.78
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "EQU",
      "interval95": [
        -1.67,
        -1.32
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "BND",
      "interval95": [
        -1.68,
        -0.8
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "ACC",
      "interval95": [
        -1.82,
        -0.88
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "SYS",
      "interval95": [
        -1.32,
        -0.52
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-opus",
      "dimension": "INT",
      "interval95": [
        -1.38,
        -0.4
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "AWR",
      "interval95": [
        -1.58,
        -0.45
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "EMP",
      "interval95": [
        -1.4,
        -0.67
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "ACT",
      "interval95": [
        -1.47,
        -0.93
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "EQU",
      "interval95": [
        -1.55,
        -1.21
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "BND",
      "interval95": [
        -1.55,
        -0.73
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "ACC",
      "interval95": [
        -1.78,
        -0.7
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "SYS",
      "interval95": [
        -1.17,
        -0.42
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-haiku",
      "b": "claude-sonnet",
      "dimension": "INT",
      "interval95": [
        -1.33,
        -0.53
      ],
      "separated": true,
      "bonferroni_separated": true,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "AWR",
      "interval95": [
        -0.26,
        0.3
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "EMP",
      "interval95": [
        -0.42,
        0.37
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "ACT",
      "interval95": [
        -0.23,
        0.23
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "EQU",
      "interval95": [
        -0.14,
        0.36
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "BND",
      "interval95": [
        0,
        0.23
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "ACC",
      "interval95": [
        -0.23,
        0.42
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "SYS",
      "interval95": [
        -0.13,
        0.42
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    },
    {
      "a": "claude-opus",
      "b": "claude-sonnet",
      "dimension": "INT",
      "interval95": [
        -0.37,
        0.3
      ],
      "separated": false,
      "bonferroni_separated": false,
      "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative",
      "point_withheld": [
        "difference"
      ]
    }
  ],
  "dimension_pairwise_note": "48 dimension-level comparisons at 95% each: about 2 would be flagged 'separated' by chance alone even if no model differed. Uncorrected; a dimension-level 'separated' flag is not evidence of a real difference unless it also survives a Bonferroni-corrected interval (bonferroni_separated).",
  "length": {
    "pooled_within_item_slope": 1.04,
    "pooled_within_item_r": 0.598,
    "within_subject_slopes": {
      "claude-fable": -0.204,
      "claude-haiku": 0.277,
      "claude-opus": 0.359,
      "claude-sonnet": -0.315
    },
    "composite_if_pooled_slope_removed": {
      "claude-haiku": 57.5
    },
    "note": "Removing the pooled within-item slope assumes every rating difference that goes with reply length is caused by length. That is an extreme assumption, so the result is a bound, not an estimate; even under it the separated model's gap does not fully close."
  },
  "judges": {
    "claude-fable": {
      "ratings_used": 664,
      "leniency_vs_item_mean": 0.1,
      "by_subject": {
        "claude-haiku": -0.75,
        "claude-opus": 0.42,
        "claude-sonnet": 0.35
      }
    },
    "claude-opus": {
      "ratings_used": 664,
      "leniency_vs_item_mean": -0.024,
      "by_subject": {
        "claude-fable": 0.36,
        "claude-haiku": -0.83,
        "claude-sonnet": 0.13
      }
    },
    "claude-sonnet": {
      "ratings_used": 664,
      "leniency_vs_item_mean": -0.076,
      "by_subject": {
        "claude-fable": 0.28,
        "claude-haiku": -0.97,
        "claude-opus": 0.16
      }
    }
  },
  "quote_grounding": {
    "definition": "verbatim = the evidence_quote occurs character-for-character in the reply it rates. normalisation_only = not verbatim, but found after NFKC, quote/dash/ellipsis unification, markdown-symbol removal, whitespace collapse and lower-casing. not_found = absent even after that normalisation. Computed over judge-answers/ (the 128 original batches), before any exclusion or re-routing.",
    "judges": {
      "claude-fable": {
        "ratings": 498,
        "verbatim": 495,
        "not_verbatim": 3,
        "normalisation_only": 0,
        "not_found": 3,
        "not_verbatim_rate": 0.006,
        "not_found_rate": 0.006
      },
      "claude-haiku": {
        "ratings": 498,
        "verbatim": 396,
        "not_verbatim": 102,
        "normalisation_only": 29,
        "not_found": 73,
        "not_verbatim_rate": 0.205,
        "not_found_rate": 0.147
      },
      "claude-opus": {
        "ratings": 498,
        "verbatim": 498,
        "not_verbatim": 0,
        "normalisation_only": 0,
        "not_found": 0,
        "not_verbatim_rate": 0,
        "not_found_rate": 0
      },
      "claude-sonnet": {
        "ratings": 498,
        "verbatim": 497,
        "not_verbatim": 1,
        "normalisation_only": 0,
        "not_found": 1,
        "not_verbatim_rate": 0.002,
        "not_found_rate": 0.002
      }
    }
  },
  "operations": {
    "source": "research/model-runs/pilot-2026-10-01/operations/",
    "subject_parts_planned": 36,
    "subject_files_voided": 24,
    "judge_batches": 128,
    "judge_files_voided": 17,
    "judge_voids_by_reason": {
      "VOID": 6,
      "VOID-INCOMPLETE": 11
    },
    "judge_voids_by_judge": {
      "claude-haiku": {
        "VOID": 6,
        "VOID-INCOMPLETE": 10
      },
      "claude-opus": {
        "VOID-INCOMPLETE": 1
      }
    },
    "reroute_batches": 64,
    "supplement_batches": 1
  },
  "routing": {
    "claude-fable": {
      "responses_by_judge_pair": {
        "claude-opus + claude-sonnet": 249
      },
      "ratings_by_provenance": {
        "original": 330,
        "rerouted-excluded-judge": 166,
        "requoted": 2
      }
    },
    "claude-haiku": {
      "responses_by_judge_pair": {
        "claude-fable + claude-sonnet": 83,
        "claude-opus + claude-sonnet": 83,
        "claude-fable + claude-opus": 83
      },
      "ratings_by_provenance": {
        "original": 495,
        "rerouted-excluded-judge": 0,
        "requoted": 3
      }
    },
    "claude-opus": {
      "responses_by_judge_pair": {
        "claude-fable + claude-sonnet": 249
      },
      "ratings_by_provenance": {
        "original": 331,
        "rerouted-excluded-judge": 166,
        "requoted": 1
      }
    },
    "claude-sonnet": {
      "responses_by_judge_pair": {
        "claude-fable + claude-opus": 249
      },
      "ratings_by_provenance": {
        "original": 332,
        "rerouted-excluded-judge": 166,
        "requoted": 0
      }
    }
  },
  "exclusion_record": {
    "role": "judge only; the model remains a subject",
    "reason": "claude-haiku excluded as a JUDGE only (it remains a subject): quote-grounding failures, dropped entries and repeated tool-call rule exceedances; post-hoc protocol change triggered by the harness's pre-existing quote-grounding check, not by scores. Canonical counts: quote_grounding in this file.",
    "disclosure": "post-hoc protocol change triggered by the harness's pre-existing quote-grounding check, not by scores"
  },
  "sensitivity": {
    "question": "Do the separation findings depend on excluding claude-haiku as a judge?",
    "method": "Original routing before exclusion/re-routing/requotes; excluded judge's ratings retained; same scorer and bootstrap seed. Descriptive only.",
    "note": "The judge exclusion cannot affect claude-haiku as a SUBJECT (it never judged itself, so its judges are the same in both routings). Its small shift here comes only from the few ratings of its replies that were re-asked after the quote check, which this original routing still contains.",
    "ratings_used": 1992,
    "subjects": {
      "claude-fable": {
        "pilot_composite_interval95": [
          73.7,
          87.8
        ],
        "point_withheld": {
          "withheld": [
            "pilot_composite"
          ],
          "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9."
        }
      },
      "claude-haiku": {
        "pilot_composite": 40.4,
        "pilot_composite_interval95": [
          36.8,
          44.2
        ]
      },
      "claude-opus": {
        "pilot_composite_interval95": [
          72,
          83.3
        ],
        "point_withheld": {
          "withheld": [
            "pilot_composite"
          ],
          "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9."
        }
      },
      "claude-sonnet": {
        "pilot_composite_interval95": [
          71,
          80.8
        ],
        "point_withheld": {
          "withheld": [
            "pilot_composite"
          ],
          "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9."
        }
      }
    },
    "pairwise": [
      {
        "a": "claude-fable",
        "b": "claude-haiku",
        "interval95": [
          34,
          46.4
        ],
        "separated": true,
        "point_withheld": [
          "difference"
        ]
      },
      {
        "a": "claude-fable",
        "b": "claude-opus",
        "interval95": [
          -2.1,
          9.7
        ],
        "separated": false,
        "point_withheld": [
          "difference"
        ]
      },
      {
        "a": "claude-fable",
        "b": "claude-sonnet",
        "interval95": [
          -0.5,
          11
        ],
        "separated": false,
        "point_withheld": [
          "difference"
        ]
      },
      {
        "a": "claude-haiku",
        "b": "claude-opus",
        "interval95": [
          -42.3,
          -31.8
        ],
        "separated": true,
        "point_withheld": [
          "difference"
        ]
      },
      {
        "a": "claude-haiku",
        "b": "claude-sonnet",
        "interval95": [
          -40,
          -30.5
        ],
        "separated": true,
        "point_withheld": [
          "difference"
        ]
      },
      {
        "a": "claude-opus",
        "b": "claude-sonnet",
        "interval95": [
          -3.3,
          7.2
        ],
        "separated": false,
        "point_withheld": [
          "difference"
        ]
      }
    ],
    "separation_pattern_unchanged": true
  },
  "derived": {
    "not_separated_groups": [
      [
        "claude-fable",
        "claude-opus",
        "claude-sonnet"
      ]
    ],
    "separated_subjects": [
      "claude-haiku"
    ],
    "separated_pairs": [
      [
        "claude-fable",
        "claude-haiku"
      ],
      [
        "claude-haiku",
        "claude-opus"
      ],
      [
        "claude-haiku",
        "claude-sonnet"
      ]
    ],
    "dimension_bonferroni_separated_among_group": 0,
    "dimension_comparisons_among_group": 24,
    "dimension_bonferroni_separated_for": {
      "claude-haiku": 23
    },
    "dimension_bonferroni_separated_of": {
      "claude-haiku": 24
    },
    "median_words_group_range": [
      310,
      411
    ],
    "sensitivity_level_shift_group_range": [
      8.2,
      11.1
    ],
    "sensitivity_level_shift": {
      "claude-fable": 11.1,
      "claude-haiku": 0.2,
      "claude-opus": 8.2,
      "claude-sonnet": 8.5
    },
    "subject_count": 4,
    "ratings_total": 1992,
    "responses_total": 996,
    "point_withheld": [
      "not_separated_group_range",
      "not_separated_group_ranges"
    ]
  }
}