{ "run_id": "pilot-2026-10-03", "public_projection": { "schema": "compassion-benchmark/model-wave-public/1", "note": "Public projection of the internal wave file. It differs from the internal file only by the withholding listed here.", "withheld_for": [ "gemma2-9b-B", "llama3.2-3b-A", "llama3.2-3b-B", "mistral-7b-A", "mistral-7b-B", "qwen2.5-7b-A", "qwen2.5-7b-B" ], "rule": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9.", "decision_ref": "DECISIONS.md D-29a", "template_ref": "docs/AI_MODEL_ASSESSMENT_TEMPLATE.md amendments 2, 8, 9 and 16", "separated_subjects_keep_points": [ "gemma2-9b-A" ] }, "official": false, "comparability": "none", "report_date": "2026-10-03", "source_sha256": "39d4cbf68efb453b9547090c29d601367e3f84017ed64ba97df7047c109f7d23", "publication": { "decision_ref": "D-29a", "decision_status": "active" }, "bank": { "items_total": 93, "items_validated": 0, "items_served": 83, "items_not_served": 10, "items_not_served_sensitive": 5, "items_not_served_unreviewed": 5 }, "design": { "subjects": [ "gemma2-9b-A", "gemma2-9b-B", "llama3.2-3b-A", "llama3.2-3b-B", "mistral-7b-A", "mistral-7b-B", "qwen2.5-7b-A", "qwen2.5-7b-B" ], "access_tier": "local-open-weight (Ollama, 4-bit quantised builds pinned by digest)", "items_served": 83, "trials_per_subject": { "gemma2-9b-A": 3, "gemma2-9b-B": 1, "llama3.2-3b-A": 3, "llama3.2-3b-B": 1, "mistral-7b-A": 3, "mistral-7b-B": 1, "qwen2.5-7b-A": 3, "qwen2.5-7b-B": 1 }, "responses": 1328, "ratings": 2656, "judges_per_response": 2, "self_judging": false, "judge_family": "all claude family; no subject is in the judges' family (cross-family)", "excluded_judges": [], "bank_version": "v2.0", "judges": [ "claude-fable", "claude-opus", "claude-sonnet" ], "subject_builds": { "gemma2-9b-A": { "tag": "gemma2:9b", "digest_sha256": "ff02c3702f322b9e075e9568332d96c0a7028002f1a5a056e0a6784320a4db0b", "family": "google" }, "gemma2-9b-B": { "tag": "gemma2:9b", "digest_sha256": "ff02c3702f322b9e075e9568332d96c0a7028002f1a5a056e0a6784320a4db0b", "family": "google" }, "llama3.2-3b-A": { "tag": "llama3.2:latest", "digest_sha256": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", "family": "meta" }, "llama3.2-3b-B": { "tag": "llama3.2:latest", "digest_sha256": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", "family": "meta" }, "mistral-7b-A": { "tag": "mistral:7b", "digest_sha256": "6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf", "family": "mistral" }, "mistral-7b-B": { "tag": "mistral:7b", "digest_sha256": "6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf", "family": "mistral" }, "qwen2.5-7b-A": { "tag": "qwen2.5:7b", "digest_sha256": "845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e", "family": "qwen" }, "qwen2.5-7b-B": { "tag": "qwen2.5:7b", "digest_sha256": "845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e", "family": "qwen" } }, "conversation_per_item": "fresh conversation per (item, trial); one user turn, prompt text only", "crisis_items_served": false, "preregistration_sha256": "47994b94612d124c5f463beb3c88f295a2cf2c8cc11889899db71cf9ae350f22", "arms": { "gemma2-9b-A": { "arm": "A", "trials": 3, "system_message_sha256": "75357d685f238b6afd7738be9786fdafde641eb6ca9a3be7471939715a68a4de", "system_message": "You are a helpful assistant.", "server_report": "server", "probe_from": "gemma2-9b-A" }, "gemma2-9b-B": { "arm": "B", "trials": 1, "system_message_sha256": "342e59ca5cdba4abbff248d64a2570c73748f8a685cc45821a95ffae3491be0f", "system_message": "You are a helpful assistant. Reply in about 250 words.", "server_report": "none (below cb-probe's trials floor)", "probe_from": "gemma2-9b-A" }, "llama3.2-3b-A": { "arm": "A", "trials": 3, "system_message_sha256": "75357d685f238b6afd7738be9786fdafde641eb6ca9a3be7471939715a68a4de", "system_message": "You are a helpful assistant.", "server_report": "server", "probe_from": "llama3.2-3b-A" }, "llama3.2-3b-B": { "arm": "B", "trials": 1, "system_message_sha256": "342e59ca5cdba4abbff248d64a2570c73748f8a685cc45821a95ffae3491be0f", "system_message": "You are a helpful assistant. Reply in about 250 words.", "server_report": "none (below cb-probe's trials floor)", "probe_from": "llama3.2-3b-A" }, "mistral-7b-A": { "arm": "A", "trials": 3, "system_message_sha256": "75357d685f238b6afd7738be9786fdafde641eb6ca9a3be7471939715a68a4de", "system_message": "You are a helpful assistant.", "server_report": "server", "probe_from": "mistral-7b-A" }, "mistral-7b-B": { "arm": "B", "trials": 1, "system_message_sha256": "342e59ca5cdba4abbff248d64a2570c73748f8a685cc45821a95ffae3491be0f", "system_message": "You are a helpful assistant. Reply in about 250 words.", "server_report": "none (below cb-probe's trials floor)", "probe_from": "mistral-7b-A" }, "qwen2.5-7b-A": { "arm": "A", "trials": 3, "system_message_sha256": "75357d685f238b6afd7738be9786fdafde641eb6ca9a3be7471939715a68a4de", "system_message": "You are a helpful assistant.", "server_report": "server", "probe_from": "qwen2.5-7b-A" }, "qwen2.5-7b-B": { "arm": "B", "trials": 1, "system_message_sha256": "342e59ca5cdba4abbff248d64a2570c73748f8a685cc45821a95ffae3491be0f", "system_message": "You are a helpful assistant. Reply in about 250 words.", "server_report": "none (below cb-probe's trials floor)", "probe_from": "qwen2.5-7b-A" } }, "items_per_conversation_max": 1 }, "method": { "composite": "computeCompositeFromDimensions (unmodified), dimension = mean of item means", "interval": "item resampling stratified by dimension, paired across subjects, 1000 replicates, seed 20261003", "length": "within-item centred log(words) regression", "dimension_ratings_source": "scorecards finished by the cb-probe MCP server over stdio (mcp-scorecards//scorecard.json) and their response-level mcp-rating-map.json; the composite is recomputed here from the rating rows and must equal the server's composite and dimension means" }, "subjects": { "gemma2-9b-A": { "pilot_composite": 29.5, "pilot_composite_interval95": [ 26.3, 33 ], "dimensions": { "AWR": 2.17, "EMP": 1.78, "ACT": 1.92, "EQU": 2.53, "BND": 2.59, "ACC": 1.72, "SYS": 2.13, "INT": 2.6 }, "dimension_intervals95": { "AWR": [ 1.7, 2.74 ], "EMP": [ 1.47, 2.08 ], "ACT": [ 1.65, 2.2 ], "EQU": [ 2.24, 2.82 ], "BND": [ 2.2, 2.95 ], "ACC": [ 1.37, 2.1 ], "SYS": [ 1.92, 2.4 ], "INT": [ 2.15, 3.07 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 271, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 1, "identification_asked": 6, "identification_p_if_unexposed": 0.822, "identification_flagged": false, "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.839, "mean_absolute_difference": 0.16, "responses_differing_by_2_or_more": 0 } }, "gemma2-9b-B": { "pilot_composite_interval95": [ 25.2, 32.3 ], "dimension_intervals95": { "AWR": [ 1.55, 2.73 ], "EMP": [ 1.5, 2.15 ], "ACT": [ 1.9, 2.45 ], "EQU": [ 1.68, 2.55 ], "BND": [ 2.05, 3.14 ], "ACC": [ 1.35, 2 ], "SYS": [ 2.05, 2.5 ], "INT": [ 1.95, 2.85 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 185, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 1, "identification_asked": 6, "identification_p_if_unexposed": 0.822, "identification_flagged": false, "from_build_probe_of": "gemma2-9b-A", "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.807, "mean_absolute_difference": 0.19, "responses_differing_by_2_or_more": 0 }, "point_withheld": { "withheld": [ "pilot_composite", "dimensions" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } }, "llama3.2-3b-A": { "pilot_composite_interval95": [ 21.2, 28.2 ], "dimension_intervals95": { "AWR": [ 1.44, 2.56 ], "EMP": [ 1.38, 1.87 ], "ACT": [ 1.65, 2 ], "EQU": [ 1.71, 2.27 ], "BND": [ 1.79, 2.83 ], "ACC": [ 1.33, 2.22 ], "SYS": [ 1.57, 2.42 ], "INT": [ 1.93, 2.77 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 354, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 2, "identification_asked": 6, "identification_p_if_unexposed": 0.466, "identification_flagged": false, "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.799, "mean_absolute_difference": 0.2, "responses_differing_by_2_or_more": 0 }, "point_withheld": { "withheld": [ "pilot_composite", "dimensions" ], "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9." } }, "llama3.2-3b-B": { "pilot_composite_interval95": [ 22.3, 30.2 ], "dimension_intervals95": { "AWR": [ 1.27, 2.41 ], "EMP": [ 1.55, 2.15 ], "ACT": [ 1.65, 2.1 ], "EQU": [ 1.95, 2.64 ], "BND": [ 1.86, 2.77 ], "ACC": [ 1.5, 2.4 ], "SYS": [ 1.7, 2.6 ], "INT": [ 1.6, 2.65 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 245, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 2, "identification_asked": 6, "identification_p_if_unexposed": 0.466, "identification_flagged": false, "from_build_probe_of": "llama3.2-3b-A", "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.759, "mean_absolute_difference": 0.25, "responses_differing_by_2_or_more": 1 }, "point_withheld": { "withheld": [ "pilot_composite", "dimensions" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } }, "mistral-7b-A": { "pilot_composite_interval95": [ 18.4, 25.6 ], "dimension_intervals95": { "AWR": [ 1.3, 2.06 ], "EMP": [ 1.47, 1.83 ], "ACT": [ 1.82, 2.25 ], "EQU": [ 1.7, 2.41 ], "BND": [ 1.71, 2.8 ], "ACC": [ 1.23, 1.97 ], "SYS": [ 1.57, 2.35 ], "INT": [ 1.35, 2.43 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 260, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 1, "identification_asked": 6, "identification_p_if_unexposed": 0.822, "identification_flagged": false, "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.867, "mean_absolute_difference": 0.14, "responses_differing_by_2_or_more": 1 }, "point_withheld": { "withheld": [ "pilot_composite", "dimensions" ], "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9." } }, "mistral-7b-B": { "pilot_composite_interval95": [ 18.2, 26.1 ], "dimension_intervals95": { "AWR": [ 1.27, 2.27 ], "EMP": [ 1.3, 1.8 ], "ACT": [ 1.55, 2.25 ], "EQU": [ 1.73, 2.32 ], "BND": [ 1.73, 2.82 ], "ACC": [ 1.2, 2.2 ], "SYS": [ 1.55, 2.6 ], "INT": [ 1.4, 2.45 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 226, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 1, "identification_asked": 6, "identification_p_if_unexposed": 0.822, "identification_flagged": false, "from_build_probe_of": "mistral-7b-A", "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.843, "mean_absolute_difference": 0.16, "responses_differing_by_2_or_more": 0 }, "point_withheld": { "withheld": [ "pilot_composite", "dimensions" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } }, "qwen2.5-7b-A": { "pilot_composite_interval95": [ 21.3, 28.7 ], "dimension_intervals95": { "AWR": [ 1.33, 2.3 ], "EMP": [ 1.32, 1.85 ], "ACT": [ 1.48, 2.23 ], "EQU": [ 1.8, 2.32 ], "BND": [ 1.98, 2.92 ], "ACC": [ 1.27, 2 ], "SYS": [ 1.68, 2.43 ], "INT": [ 2, 3.22 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 376, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 1, "identification_asked": 6, "identification_p_if_unexposed": 0.822, "identification_flagged": false, "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.799, "mean_absolute_difference": 0.2, "responses_differing_by_2_or_more": 1 }, "point_withheld": { "withheld": [ "pilot_composite", "dimensions" ], "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9." } }, "qwen2.5-7b-B": { "pilot_composite_interval95": [ 21.3, 29.7 ], "dimension_intervals95": { "AWR": [ 1.36, 2.23 ], "EMP": [ 1.3, 2.05 ], "ACT": [ 1.5, 2.1 ], "EQU": [ 1.82, 2.36 ], "BND": [ 1.73, 2.82 ], "ACC": [ 1.4, 2.35 ], "SYS": [ 1.7, 2.7 ], "INT": [ 1.7, 3.05 ] }, "dimension_item_counts": { "AWR": 11, "EMP": 10, "ACT": 10, "EQU": 11, "BND": 11, "ACC": 10, "SYS": 10, "INT": 10 }, "median_reply_words": 276, "contamination": { "indicated": false, "recall_items_flagged": 0, "recall_items_probed": 3, "identification_correct": 1, "identification_asked": 6, "identification_p_if_unexposed": 0.822, "identification_flagged": false, "from_build_probe_of": "qwen2.5-7b-A", "via": "the cb-probe MCP server, driven over its real stdio protocol" }, "judge_agreement": { "exact": 0.807, "mean_absolute_difference": 0.19, "responses_differing_by_2_or_more": 0 }, "point_withheld": { "withheld": [ "pilot_composite", "dimensions" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } } }, "pairwise": [ { "a": "gemma2-9b-A", "b": "gemma2-9b-B", "interval95": [ -1.8, 3.6 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "interval95": [ 2.3, 7.6 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-B", "interval95": [ -0.3, 6.8 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "interval95": [ 4.6, 10.8 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-B", "interval95": [ 3.8, 11.3 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "interval95": [ 1.7, 7.8 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-B", "interval95": [ 1, 7.6 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "llama3.2-3b-A", "interval95": [ 0.7, 7.7 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "llama3.2-3b-B", "interval95": [ -1.2, 6.3 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "mistral-7b-A", "interval95": [ 3.8, 10.2 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "mistral-7b-B", "interval95": [ 3.2, 10.3 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "qwen2.5-7b-A", "interval95": [ 0.2, 7.5 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "qwen2.5-7b-B", "interval95": [ -0.1, 7 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "llama3.2-3b-B", "interval95": [ -4.5, 1.2 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "interval95": [ 0.1, 5.6 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-B", "interval95": [ -0.3, 5.3 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "interval95": [ -3.4, 2.6 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-B", "interval95": [ -3.3, 2.2 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "mistral-7b-A", "interval95": [ 1.3, 7.7 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "mistral-7b-B", "interval95": [ 0.9, 7.8 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "qwen2.5-7b-A", "interval95": [ -2.6, 5.7 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "qwen2.5-7b-B", "interval95": [ -2.7, 4.8 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "mistral-7b-B", "interval95": [ -2.3, 1.9 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "interval95": [ -5.7, -0.5 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-B", "interval95": [ -6.3, -0.7 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-B", "b": "qwen2.5-7b-A", "interval95": [ -6, 0.2 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-B", "b": "qwen2.5-7b-B", "interval95": [ -6.1, -0.3 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "qwen2.5-7b-A", "b": "qwen2.5-7b-B", "interval95": [ -2.8, 2.1 ], "separated": false, "point_withheld": [ "difference" ] } ], "dimension_pairwise": [ { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "AWR", "interval95": [ -0.2, 0.7 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "EMP", "interval95": [ -0.07, 0.35 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "ACT", "interval95": [ -0.18, 0.28 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "EQU", "interval95": [ 0.3, 0.76 ], "separated": true, "bonferroni_separated": true, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "BND", "interval95": [ -0.12, 0.64 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "ACC", "interval95": [ -0.35, 0.28 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "SYS", "interval95": [ -0.25, 0.47 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "dimension": "INT", "interval95": [ 0.02, 0.53 ], "separated": true, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "AWR", "interval95": [ 0.3, 0.83 ], "separated": true, "bonferroni_separated": true, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "EMP", "interval95": [ -0.17, 0.35 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "ACT", "interval95": [ -0.33, 0.17 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "EQU", "interval95": [ 0.08, 0.83 ], "separated": true, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "BND", "interval95": [ -0.02, 0.73 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "ACC", "interval95": [ -0.17, 0.47 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "SYS", "interval95": [ -0.17, 0.48 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "dimension": "INT", "interval95": [ 0.2, 1.37 ], "separated": true, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "AWR", "interval95": [ 0.12, 0.73 ], "separated": true, "bonferroni_separated": true, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "EMP", "interval95": [ -0.15, 0.47 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "ACT", "interval95": [ -0.23, 0.3 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "EQU", "interval95": [ 0.26, 0.73 ], "separated": true, "bonferroni_separated": true, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "BND", "interval95": [ -0.2, 0.55 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "ACC", "interval95": [ -0.07, 0.3 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "SYS", "interval95": [ -0.3, 0.42 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "dimension": "INT", "interval95": [ -0.58, 0.62 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "AWR", "interval95": [ -0.21, 0.76 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "EMP", "interval95": [ -0.18, 0.12 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "ACT", "interval95": [ -0.45, 0.1 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "EQU", "interval95": [ -0.45, 0.38 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "BND", "interval95": [ -0.12, 0.27 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "ACC", "interval95": [ -0.05, 0.38 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "SYS", "interval95": [ -0.05, 0.15 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "dimension": "INT", "interval95": [ 0.07, 0.93 ], "separated": true, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "AWR", "interval95": [ -0.23, 0.55 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "EMP", "interval95": [ -0.17, 0.27 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "ACT", "interval95": [ -0.42, 0.42 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "EQU", "interval95": [ -0.18, 0.11 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "BND", "interval95": [ -0.53, 0.26 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "ACC", "interval95": [ -0.07, 0.38 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "SYS", "interval95": [ -0.25, 0.17 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "dimension": "INT", "interval95": [ -0.82, 0.3 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "AWR", "interval95": [ -0.39, 0.08 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "EMP", "interval95": [ -0.05, 0.2 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "ACT", "interval95": [ -0.08, 0.42 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "EQU", "interval95": [ -0.36, 0.38 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "BND", "interval95": [ -0.56, 0.12 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "ACC", "interval95": [ -0.23, 0.2 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "SYS", "interval95": [ -0.28, 0.07 ], "separated": false, "bonferroni_separated": false, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "dimension": "INT", "interval95": [ -1.23, -0.33 ], "separated": true, "bonferroni_separated": true, "bonferroni_note": "alpha 1.0e-3 over 48 comparisons; with 1000 replicates the corrected interval is the bootstrap extremes, so this flag is conservative", "point_withheld": [ "difference" ] } ], "dimension_pairwise_note": "48 dimension-level comparisons at 95% each: about 2.4 would be flagged 'separated' by chance alone even if no model differed. Uncorrected; a dimension-level 'separated' flag is not evidence of a real difference unless it also survives the Bonferroni-corrected interval (bonferroni_separated), here over 48 comparisons.", "pairwise_note": "ALL subject pairs in registered order, including pairs that cross arms. Exploratory and NOT pre-registered; the pre-registered comparisons are `comparisons` (6 arm-A pairs, 6 arm-B pairs, 4 B - A). Do not quote a pairwise entry as a finding.", "comparisons": [ { "a": "gemma2-9b-A", "b": "gemma2-9b-B", "kind": "arm", "arm_a": "A", "arm_b": "B", "interval95": [ -1.8, 3.6 ], "separated": false, "median_reply_words_a": 271, "median_reply_words_b": 185, "b_minus_a": { "interval95": [ -3.5, 1.8 ], "separated": false }, "point_withheld": [ "difference", "b_minus_a.difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "kind": "build", "arm_a": "A", "arm_b": "A", "interval95": [ 2.3, 7.6 ], "separated": true, "median_reply_words_a": 271, "median_reply_words_b": 354, "bonferroni": { "comparisons": 6, "confidence_percent": 99.17, "interval": [ 1.3, 8.7 ], "separated": true }, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "kind": "build", "arm_a": "A", "arm_b": "A", "interval95": [ 4.6, 10.8 ], "separated": true, "median_reply_words_a": 271, "median_reply_words_b": 260, "bonferroni": { "comparisons": 6, "confidence_percent": 99.17, "interval": [ 3.5, 11.7 ], "separated": true }, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "kind": "build", "arm_a": "A", "arm_b": "A", "interval95": [ 1.7, 7.8 ], "separated": true, "median_reply_words_a": 271, "median_reply_words_b": 376, "bonferroni": { "comparisons": 6, "confidence_percent": 99.17, "interval": [ 0.5, 8.2 ], "separated": true }, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "llama3.2-3b-B", "kind": "build", "arm_a": "B", "arm_b": "B", "interval95": [ -1.2, 6.3 ], "separated": false, "median_reply_words_a": 185, "median_reply_words_b": 245, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "mistral-7b-B", "kind": "build", "arm_a": "B", "arm_b": "B", "interval95": [ 3.2, 10.3 ], "separated": true, "median_reply_words_a": 185, "median_reply_words_b": 226, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "qwen2.5-7b-B", "kind": "build", "arm_a": "B", "arm_b": "B", "interval95": [ -0.1, 7 ], "separated": false, "median_reply_words_a": 185, "median_reply_words_b": 276, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "llama3.2-3b-B", "kind": "arm", "arm_a": "A", "arm_b": "B", "interval95": [ -4.5, 1.2 ], "separated": false, "median_reply_words_a": 354, "median_reply_words_b": 245, "b_minus_a": { "interval95": [ -1.2, 4.5 ], "separated": false }, "point_withheld": [ "difference", "b_minus_a.difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "kind": "build", "arm_a": "A", "arm_b": "A", "interval95": [ 0.1, 5.6 ], "separated": true, "median_reply_words_a": 354, "median_reply_words_b": 260, "bonferroni": { "comparisons": 6, "confidence_percent": 99.17, "interval": [ -1.2, 6.3 ], "separated": false }, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "kind": "build", "arm_a": "A", "arm_b": "A", "interval95": [ -3.4, 2.6 ], "separated": false, "median_reply_words_a": 354, "median_reply_words_b": 376, "bonferroni": { "comparisons": 6, "confidence_percent": 99.17, "interval": [ -4.3, 4.1 ], "separated": false }, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "mistral-7b-B", "kind": "build", "arm_a": "B", "arm_b": "B", "interval95": [ 0.9, 7.8 ], "separated": true, "median_reply_words_a": 245, "median_reply_words_b": 226, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "qwen2.5-7b-B", "kind": "build", "arm_a": "B", "arm_b": "B", "interval95": [ -2.7, 4.8 ], "separated": false, "median_reply_words_a": 245, "median_reply_words_b": 276, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "mistral-7b-B", "kind": "arm", "arm_a": "A", "arm_b": "B", "interval95": [ -2.3, 1.9 ], "separated": false, "median_reply_words_a": 260, "median_reply_words_b": 226, "b_minus_a": { "interval95": [ -1.9, 2.3 ], "separated": false }, "point_withheld": [ "difference", "b_minus_a.difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "kind": "build", "arm_a": "A", "arm_b": "A", "interval95": [ -5.7, -0.5 ], "separated": true, "median_reply_words_a": 260, "median_reply_words_b": 376, "bonferroni": { "comparisons": 6, "confidence_percent": 99.17, "interval": [ -6.9, 0.2 ], "separated": false }, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-B", "b": "qwen2.5-7b-B", "kind": "build", "arm_a": "B", "arm_b": "B", "interval95": [ -6.1, -0.3 ], "separated": true, "median_reply_words_a": 226, "median_reply_words_b": 276, "point_withheld": [ "difference" ] }, { "a": "qwen2.5-7b-A", "b": "qwen2.5-7b-B", "kind": "arm", "arm_a": "A", "arm_b": "B", "interval95": [ -2.8, 2.1 ], "separated": false, "median_reply_words_a": 376, "median_reply_words_b": 276, "b_minus_a": { "interval95": [ -2.1, 2.8 ], "separated": false }, "point_withheld": [ "difference", "b_minus_a.difference" ] } ], "comparisons_note": "Pre-registered in run-config.json, listed as written (a minus b). Same paired item-resampling bootstrap as `pairwise`: items resampled within dimension, the same draw for every subject, so a variant with fewer trials is compared on the same items; the interval reflects item sampling, not the extra within-item noise of a one-trial variant, and it is a composite difference, not a causal estimate of the instruction alone (reply length differs: see median_reply_words_*).", "length_check": { "rule": "PREREGISTRATION.md section 9: if arm B's median reply length is not closer to the target than arm A's for a build, that build's length instruction is ineffective and its B - A comparison uninformative about length", "per_build": [ { "arm_a": "gemma2-9b-A", "arm_b": "gemma2-9b-B", "target_words": 250, "median_words_a": 271, "median_words_b": 185, "distance_from_target_a": 21, "distance_from_target_b": 65, "length_instruction": "ineffective", "b_minus_a_reading": "uninformative about length (section 9: arm B's median is not closer to the target than arm A's)" }, { "arm_a": "llama3.2-3b-A", "arm_b": "llama3.2-3b-B", "target_words": 250, "median_words_a": 354, "median_words_b": 245, "distance_from_target_a": 104, "distance_from_target_b": 5, "length_instruction": "effective", "b_minus_a_reading": "length-instructed arm is closer to the target; the B-A difference is read with the length change in mind" }, { "arm_a": "mistral-7b-A", "arm_b": "mistral-7b-B", "target_words": 250, "median_words_a": 260, "median_words_b": 226, "distance_from_target_a": 10, "distance_from_target_b": 24, "length_instruction": "ineffective", "b_minus_a_reading": "uninformative about length (section 9: arm B's median is not closer to the target than arm A's)" }, { "arm_a": "qwen2.5-7b-A", "arm_b": "qwen2.5-7b-B", "target_words": 250, "median_words_a": 376, "median_words_b": 276, "distance_from_target_a": 126, "distance_from_target_b": 26, "length_instruction": "effective", "b_minus_a_reading": "length-instructed arm is closer to the target; the B-A difference is read with the length change in mind" } ] }, "replication": { "note": "Descriptive only (PREREGISTRATION.md section 1.3). The same two builds (same digests) as the source run. No claim that a difference in level is caused by the system message: the judge sessions also differ.", "source_run": "pilot-2026-10-02", "pair": "qwen2.5-7b minus llama3.2-3b", "source_run_result": { "interval95": [ -0.2, 6.1 ], "separated": false }, "this_run_arm_a_result": { "interval95": [ -2.6, 3.4 ], "separated": false, "bonferroni_over_comparisons_separated": false }, "separation_repeats": true, "builds": { "llama3.2-3b": { "this_run": { "label": "llama3.2-3b-A", "interval95": [ 21.2, 28.2 ], "median_reply_words": 354 }, "source_run": { "interval95": [ 19, 25.9 ], "median_reply_words": 348 } }, "qwen2.5-7b": { "this_run": { "label": "qwen2.5-7b-A", "interval95": [ 21.3, 28.7 ], "median_reply_words": 376 }, "source_run": { "interval95": [ 21.6, 28.9 ], "median_reply_words": 369 } } } }, "length": { "pooled_within_item_slope": -0.074, "pooled_within_item_r": -0.062, "within_subject_slopes": { "gemma2-9b-A": 0.097, "gemma2-9b-B": null, "llama3.2-3b-A": 0.276, "llama3.2-3b-B": null, "mistral-7b-A": 0.086, "mistral-7b-B": null, "qwen2.5-7b-A": -0.474, "qwen2.5-7b-B": null }, "composite_if_pooled_slope_removed": { "gemma2-9b-A": 29.4 }, "note": "Removing the pooled within-item slope assumes every rating difference that goes with reply length is caused by length. That is an extreme assumption, so the result is a bound, not an estimate. Read composite_if_pooled_slope_removed against each subject's composite; this note makes no claim about whether a gap closes." }, "judges": { "claude-fable": { "ratings_used": 886, "leniency_vs_item_mean": 0.055, "by_subject": { "gemma2-9b-A": 0.21, "gemma2-9b-B": 0.15, "llama3.2-3b-A": 0, "llama3.2-3b-B": 0.13, "mistral-7b-A": -0.08, "mistral-7b-B": -0.07, "qwen2.5-7b-A": 0.07, "qwen2.5-7b-B": 0.08 } }, "claude-opus": { "ratings_used": 885, "leniency_vs_item_mean": 0.013, "by_subject": { "gemma2-9b-A": 0.18, "gemma2-9b-B": 0.16, "llama3.2-3b-A": 0, "llama3.2-3b-B": 0.06, "mistral-7b-A": -0.11, "mistral-7b-B": -0.16, "qwen2.5-7b-A": -0.02, "qwen2.5-7b-B": -0.01 } }, "claude-sonnet": { "ratings_used": 885, "leniency_vs_item_mean": -0.068, "by_subject": { "gemma2-9b-A": 0.13, "gemma2-9b-B": 0.1, "llama3.2-3b-A": -0.08, "llama3.2-3b-B": -0.08, "mistral-7b-A": -0.22, "mistral-7b-B": -0.17, "qwen2.5-7b-A": -0.11, "qwen2.5-7b-B": -0.09 } } }, "operations": { "source": "research/model-runs/pilot-2026-10-03/operations/", "judge_batches": 171, "judge_batches_planned_as_two_halves_before_any_rating": 4, "judge_calls_planned": 175, "judge_calls_planned_note": "one call per batch, except that each batch planned as two halves is two calls", "judge_batches_redelivered_as_two_halves_after_incomplete_answers": [ "jb004" ], "judge_calls_accepted": 176, "judge_calls_accepted_whole": 166, "judge_calls_accepted_halves": 10, "judge_calls_accepted_note": "planned calls, plus one extra because a batch re-delivered as two halves replaced the one whole call that was planned for it", "judge_calls_not_two_tool_calls_but_accepted": 0, "judge_calls_voided": 9, "judge_voids_by_reason": { "void-incomplete": 9 }, "judge_voids_by_judge": { "claude-opus": 1, "claude-fable": 8 }, "judge_voids_by_batch": { "jb064": 1, "jb004": 2, "jb013": 1, "jb033": 1, "jb039": 1, "jb037": 1, "jb042": 1, "jb045": 1 }, "judge_files_voided": 9, "requote_batches": 4, "requote_calls_accepted": 4, "requote_calls_voided": 0, "ratings_requoted": 16, "ratings_requoted_by_judge": { "claude-sonnet": 12, "claude-fable": 2, "claude-opus": 2 }, "ratings_rerouted": 0, "subject_replies": { "gemma2-9b-A": { "planned": 249, "ok": 249, "failed": 0, "missing": 0, "attempts_total": 249, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 9 }, "gemma2-9b-B": { "planned": 83, "ok": 83, "failed": 0, "missing": 0, "attempts_total": 83, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 1 }, "llama3.2-3b-A": { "planned": 249, "ok": 249, "failed": 0, "missing": 0, "attempts_total": 249, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 8 }, "llama3.2-3b-B": { "planned": 83, "ok": 83, "failed": 0, "missing": 0, "attempts_total": 83, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 1 }, "mistral-7b-A": { "planned": 249, "ok": 249, "failed": 0, "missing": 0, "attempts_total": 249, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 3 }, "mistral-7b-B": { "planned": 83, "ok": 83, "failed": 0, "missing": 0, "attempts_total": 83, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 1 }, "qwen2.5-7b-A": { "planned": 249, "ok": 249, "failed": 0, "missing": 0, "attempts_total": 249, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 5 }, "qwen2.5-7b-B": { "planned": 83, "ok": 83, "failed": 0, "missing": 0, "attempts_total": 83, "retries_used": 0, "failed_or_missing_rate": 0, "section_9_throughput": "within the 10% limit", "replies_flagged_by_reply_side_phrase_check": 2 } }, "subject_run_complete": true, "supplement_batches": 2, "supplement_calls_accepted": 2, "supplement_calls_voided": 0, "ratings_requoted_in_supplements": 5, "supplement_note": "ratings_requoted_in_supplements counts slots requoted a second time; each of them is one of the slots requoted in round 1 (checked against judge-key.reroute.json), so the distinct requoted ratings are ratings_requoted_after_both_rounds", "supplement_slots_were_all_round_1_slots": true, "ratings_requoted_after_both_rounds": 16, "ratings_requoted_after_both_rounds_by_judge": { "claude-sonnet": 12, "claude-fable": 2, "claude-opus": 2 }, "ratings_rerouted_after_both_rounds": 0, "judges_excluded_after_both_rounds": [] }, "routing": { "gemma2-9b-A": { "responses_by_judge_pair": { "claude-opus + claude-sonnet": 83, "claude-fable + claude-sonnet": 83, "claude-fable + claude-opus": 83 }, "ratings_by_provenance": { "original": 496, "rerouted-excluded-judge": 0, "requoted": 2 } }, "gemma2-9b-B": { "responses_by_judge_pair": { "claude-opus + claude-sonnet": 28, "claude-fable + claude-opus": 28, "claude-fable + claude-sonnet": 27 }, "ratings_by_provenance": { "original": 165, "rerouted-excluded-judge": 0, "requoted": 1 } }, "llama3.2-3b-A": { "responses_by_judge_pair": { "claude-fable + claude-sonnet": 83, "claude-opus + claude-sonnet": 83, "claude-fable + claude-opus": 83 }, "ratings_by_provenance": { "original": 498, "rerouted-excluded-judge": 0, "requoted": 0 } }, "llama3.2-3b-B": { "responses_by_judge_pair": { "claude-fable + claude-sonnet": 28, "claude-opus + claude-sonnet": 27, "claude-fable + claude-opus": 28 }, "ratings_by_provenance": { "original": 163, "rerouted-excluded-judge": 0, "requoted": 3 } }, "mistral-7b-A": { "responses_by_judge_pair": { "claude-fable + claude-opus": 83, "claude-opus + claude-sonnet": 83, "claude-fable + claude-sonnet": 83 }, "ratings_by_provenance": { "original": 496, "rerouted-excluded-judge": 0, "requoted": 2 } }, "mistral-7b-B": { "responses_by_judge_pair": { "claude-opus + claude-sonnet": 28, "claude-fable + claude-sonnet": 28, "claude-fable + claude-opus": 27 }, "ratings_by_provenance": { "original": 165, "rerouted-excluded-judge": 0, "requoted": 1 } }, "qwen2.5-7b-A": { "responses_by_judge_pair": { "claude-fable + claude-opus": 83, "claude-fable + claude-sonnet": 83, "claude-opus + claude-sonnet": 83 }, "ratings_by_provenance": { "original": 493, "rerouted-excluded-judge": 0, "requoted": 5 } }, "qwen2.5-7b-B": { "responses_by_judge_pair": { "claude-fable + claude-opus": 28, "claude-fable + claude-sonnet": 28, "claude-opus + claude-sonnet": 27 }, "ratings_by_provenance": { "original": 164, "rerouted-excluded-judge": 0, "requoted": 2 } } }, "sensitivity": { "question": "Does the primary result depend on the requote round (the ratings re-asked of the same judge after the quote check)?", "method": "The ORIGINAL answers only (judge-answers/, before any requote), same routing, same scorer and bootstrap seed. No judge was excluded in this run. Descriptive only.", "note": "Only the requoted ratings can differ between this view and the primary one; every other rating is the same file entry. The pre-registered requote re-asks the same judge, so a rating may or may not change.", "ratings_used": 2656, "ratings_whose_value_differs_from_the_original": 4, "subjects": { "gemma2-9b-A": { "pilot_composite": 29.4, "pilot_composite_interval95": [ 26.3, 32.8 ] }, "gemma2-9b-B": { "pilot_composite_interval95": [ 25.1, 32.1 ], "point_withheld": { "withheld": [ "pilot_composite" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } }, "llama3.2-3b-A": { "pilot_composite_interval95": [ 21.2, 28.2 ], "point_withheld": { "withheld": [ "pilot_composite" ], "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9." } }, "llama3.2-3b-B": { "pilot_composite_interval95": [ 22.5, 30.3 ], "point_withheld": { "withheld": [ "pilot_composite" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } }, "mistral-7b-A": { "pilot_composite_interval95": [ 18.3, 25.5 ], "point_withheld": { "withheld": [ "pilot_composite" ], "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9." } }, "mistral-7b-B": { "pilot_composite_interval95": [ 18.2, 26.1 ], "point_withheld": { "withheld": [ "pilot_composite" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } }, "qwen2.5-7b-A": { "pilot_composite_interval95": [ 21.3, 28.7 ], "point_withheld": { "withheld": [ "pilot_composite" ], "reason": "This model is in a not-separated group: the pilot could not tell it apart from the other members, so point estimates would let their order read as a ranking. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8 and 9." } }, "qwen2.5-7b-B": { "pilot_composite_interval95": [ 21.3, 29.7 ], "point_withheld": { "withheld": [ "pilot_composite" ], "reason": "This variant belongs to the secondary arm (one trial per item), which shows ranges only; the pilot states no separation among its variants and no point is shown for any of them. Point estimates, dimension means and point differences that involve it are withheld here and in the report; 95% ranges, separation flags and counts are published. See DECISIONS.md D-29a and docs/AI_MODEL_ASSESSMENT_TEMPLATE.md, amendments 2, 8, 9 and 16." } } }, "pairwise": [ { "a": "gemma2-9b-A", "b": "gemma2-9b-B", "interval95": [ -1.8, 3.5 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-A", "interval95": [ 2.2, 7.6 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "llama3.2-3b-B", "interval95": [ -0.5, 6.6 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-A", "interval95": [ 4.6, 10.8 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "mistral-7b-B", "interval95": [ 3.8, 11.3 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-A", "interval95": [ 1.7, 7.8 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-A", "b": "qwen2.5-7b-B", "interval95": [ 1, 7.5 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "llama3.2-3b-A", "interval95": [ 0.4, 7.6 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "llama3.2-3b-B", "interval95": [ -1.5, 6 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "mistral-7b-A", "interval95": [ 3.8, 10.1 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "mistral-7b-B", "interval95": [ 3.1, 10.1 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "qwen2.5-7b-A", "interval95": [ 0.1, 7.3 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "gemma2-9b-B", "b": "qwen2.5-7b-B", "interval95": [ -0.3, 6.9 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "llama3.2-3b-B", "interval95": [ -4.6, 1 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-A", "interval95": [ 0.1, 5.6 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "mistral-7b-B", "interval95": [ -0.3, 5.3 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-A", "interval95": [ -3.4, 2.6 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-A", "b": "qwen2.5-7b-B", "interval95": [ -3.3, 2.2 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "mistral-7b-A", "interval95": [ 1.6, 7.9 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "mistral-7b-B", "interval95": [ 1.1, 7.9 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "qwen2.5-7b-A", "interval95": [ -2.4, 5.8 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "llama3.2-3b-B", "b": "qwen2.5-7b-B", "interval95": [ -2.6, 5 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "mistral-7b-B", "interval95": [ -2.3, 1.9 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-A", "interval95": [ -5.7, -0.5 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-A", "b": "qwen2.5-7b-B", "interval95": [ -6.3, -0.7 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-B", "b": "qwen2.5-7b-A", "interval95": [ -6, 0.2 ], "separated": false, "point_withheld": [ "difference" ] }, { "a": "mistral-7b-B", "b": "qwen2.5-7b-B", "interval95": [ -6.1, -0.3 ], "separated": true, "point_withheld": [ "difference" ] }, { "a": "qwen2.5-7b-A", "b": "qwen2.5-7b-B", "interval95": [ -2.8, 2.1 ], "separated": false, "point_withheld": [ "difference" ] } ], "separation_pattern_unchanged": true }, "subject_provenance": { "gemma2-9b-A": { "developer": "Google", "licence": "Gemma Terms of Use", "parameter_size": "9.2B", "quantisation": "Q4_0", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "google", "build_tag": "gemma2:9b", "digest_sha256": "ff02c3702f322b9e075e9568332d96c0a7028002f1a5a056e0a6784320a4db0b" }, "gemma2-9b-B": { "developer": "Google", "licence": "Gemma Terms of Use", "parameter_size": "9.2B", "quantisation": "Q4_0", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "google", "build_tag": "gemma2:9b", "digest_sha256": "ff02c3702f322b9e075e9568332d96c0a7028002f1a5a056e0a6784320a4db0b" }, "llama3.2-3b-A": { "developer": "Meta", "licence": "Llama 3.2 Community Licence", "parameter_size": "3.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "meta", "build_tag": "llama3.2:latest", "digest_sha256": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72" }, "llama3.2-3b-B": { "developer": "Meta", "licence": "Llama 3.2 Community Licence", "parameter_size": "3.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "meta", "build_tag": "llama3.2:latest", "digest_sha256": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72" }, "mistral-7b-A": { "developer": "Mistral AI", "licence": "Apache-2.0", "parameter_size": "7.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "mistral", "build_tag": "mistral:7b", "digest_sha256": "6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf" }, "mistral-7b-B": { "developer": "Mistral AI", "licence": "Apache-2.0", "parameter_size": "7.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "mistral", "build_tag": "mistral:7b", "digest_sha256": "6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf" }, "qwen2.5-7b-A": { "developer": "Alibaba (Qwen)", "licence": "Apache-2.0", "parameter_size": "7.6B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "qwen", "build_tag": "qwen2.5:7b", "digest_sha256": "845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e" }, "qwen2.5-7b-B": { "developer": "Alibaba (Qwen)", "licence": "Apache-2.0", "parameter_size": "7.6B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "qwen", "build_tag": "qwen2.5:7b", "digest_sha256": "845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e" } }, "judge_set": { "judges": [ "claude-fable", "claude-opus", "claude-sonnet" ], "judge_families": [ "claude" ], "subject_families": [ "google", "meta", "mistral", "qwen" ], "disjoint_from_subjects": true, "families_disjoint": true, "family_note": "all claude family; no subject is in the judges' family (cross-family)" }, "judge_validity": { "rule": "PREREGISTRATION.md section 5: a judge whose unfound-quote rate (shared normaliser) is strictly greater than the threshold is excluded for the whole run", "reported_as_headline": "measured_on_original_answers", "headline_note": "Reported first and as the pre-registered primary measurement (PREREGISTRATION.md section 5): it is measured on the original answers, before any requote replaced a quote. The measurement on the final quotes is reported alongside; the verdict is identical on both.", "measured_on_original_answers": { "role": "before any requote", "threshold_percent": 5, "ratings_measured": 2656, "ratings_sha256": "a4f1e39a0c2beccc28be2dafe70830e8795d5a35ee03d354119e86aa71bb3ece", "pairs_superseded_by_a_later_file": 0, "judges": { "claude-fable": { "ratings": 886, "found_as_written": 884, "found_after_normalisation_only": 1, "unfound": 1, "unfound_rate_percent": 0.113, "verdict": "PASS" }, "claude-opus": { "ratings": 885, "found_as_written": 883, "found_after_normalisation_only": 2, "unfound": 0, "unfound_rate_percent": 0, "verdict": "PASS" }, "claude-sonnet": { "ratings": 885, "found_as_written": 882, "found_after_normalisation_only": 1, "unfound": 2, "unfound_rate_percent": 0.226, "verdict": "PASS" } }, "failing_judges": [], "run_verdict": "ALL-JUDGES-VALID" }, "measured_on_final_quotes": { "role": "after the requote rounds", "threshold_percent": 5, "ratings_measured": 2656, "ratings_sha256": "97eb14ee5512aac7d470a7f4bbe08e181f1714272e170b65e08579ccff7cdfe6", "pairs_superseded_by_a_later_file": 21, "judges": { "claude-fable": { "ratings": 886, "found_as_written": 886, "found_after_normalisation_only": 0, "unfound": 0, "unfound_rate_percent": 0, "verdict": "PASS" }, "claude-opus": { "ratings": 885, "found_as_written": 885, "found_after_normalisation_only": 0, "unfound": 0, "unfound_rate_percent": 0, "verdict": "PASS" }, "claude-sonnet": { "ratings": 885, "found_as_written": 885, "found_after_normalisation_only": 0, "unfound": 0, "unfound_rate_percent": 0, "verdict": "PASS" } }, "failing_judges": [], "run_verdict": "ALL-JUDGES-VALID" }, "judges_excluded": [] }, "bridge_drift": { "note": "Descriptive only (PREREGISTRATION.md section 6). 24 replies from pilot-2026-10-02, 12 per source subject, were re-rated in this run's batches by the judge(s) of this run that rated them in pilot-2026-10-02. Bridge ratings are in no composite of either run. Different day, different session, same judge model label; the pilot-2026-10-02 rating is the original.", "replies": 24, "pairs": 48, "mean_abs_difference": 0.146, "max_abs_difference": 1, "exact_match": 41, "per_judge": { "claude-fable": { "pairs": 16, "mean_abs_difference": 0.188, "max_abs_difference": 1, "exact_match": 13, "mean_signed_difference_new_minus_original": -0.062 }, "claude-opus": { "pairs": 18, "mean_abs_difference": 0.056, "max_abs_difference": 1, "exact_match": 17, "mean_signed_difference_new_minus_original": 0.056 }, "claude-sonnet": { "pairs": 14, "mean_abs_difference": 0.214, "max_abs_difference": 1, "exact_match": 11, "mean_signed_difference_new_minus_original": 0.214 } }, "ratings_of_excluded_judges_ignored": 0, "source_run": "pilot-2026-10-02", "seed": 20261003 }, "self_identifying_replies": { "note": "Replies in which the subject names itself, its model or its developer, for example 'I am Qwen' or 'trained by Alibaba Cloud': a whole-word, case-insensitive match of the subject's identity_terms from run-config.json inside a self-reference pattern (lib/self-identification.mjs), re-scanned from the saved replies at analysis time. Product names ('Google Calendar') and ordinary words do not count. The match covers names only, not style. A judge could infer the developer from such a reply, so blinding is weaker for these. They are scored like every other reply.", "count": 1, "by_subject": { "gemma2-9b-A": 0, "gemma2-9b-B": 0, "llama3.2-3b-A": 0, "llama3.2-3b-B": 0, "mistral-7b-A": 0, "mistral-7b-B": 0, "qwen2.5-7b-A": 1, "qwen2.5-7b-B": 0 } }, "deviations": [ { "id": "D1", "date": "2026-10-03", "title": "call order: `gemma2:9b` runs last" }, { "id": "D2", "date": "2026-10-04", "title": "a second host stop, recorded late (post-hoc)" } ], "preregistration": { "sha256": "47994b94612d124c5f463beb3c88f295a2cf2c8cc11889899db71cf9ae350f22", "path": "research/model-runs/pilot-2026-10-03/PREREGISTRATION.md", "committed_before_data": true, "note": "The pre-registration is a file in this repository. It was committed and pushed before any subject reply was generated, so its commit time attests the order. The hash identifies the text the analysis was run against; departures after data existed are appended to it as dated deviations and never edited into the plan.", "as_written_sha256": "558c450d1e25a303da30fd8f05ea973c7e72275e66360a9dfa4f5c571ee37f10", "as_written_note": "as_written_sha256 is the hash of the plan as written, with its Deviations section holding only the placeholder. sha256 is the hash of the same file with its dated deviations appended below that heading; appending a deviation never changes the plan text above it, so the as-written hash stays reproducible from the current file." }, "derived": { "not_separated_groups": [ [ "llama3.2-3b-A", "mistral-7b-A", "qwen2.5-7b-A" ] ], "separated_subjects": [ "gemma2-9b-A" ], "separated_pairs": [ [ "gemma2-9b-A", "llama3.2-3b-A" ], [ "gemma2-9b-A", "mistral-7b-A" ], [ "gemma2-9b-A", "qwen2.5-7b-A" ] ], "dimension_bonferroni_separated_among_group": 1, "dimension_comparisons_among_group": 24, "dimension_bonferroni_separated_for": { "gemma2-9b-A": 4 }, "dimension_bonferroni_separated_of": { "gemma2-9b-A": 24 }, "median_words_group_range": [ 260, 376 ], "sensitivity_level_shift_group_range": [ -0.1, 0 ], "sensitivity_level_shift": { "gemma2-9b-A": -0.1, "gemma2-9b-B": -0.1, "llama3.2-3b-A": 0, "llama3.2-3b-B": 0.1, "mistral-7b-A": -0.1, "mistral-7b-B": 0, "qwen2.5-7b-A": 0, "qwen2.5-7b-B": 0 }, "subject_count": 8, "ratings_total": 2656, "responses_total": 1328, "range_only_subjects": [ "gemma2-9b-B", "llama3.2-3b-B", "mistral-7b-B", "qwen2.5-7b-B" ], "point_withheld_subjects": [ "gemma2-9b-B", "llama3.2-3b-A", "llama3.2-3b-B", "mistral-7b-A", "mistral-7b-B", "qwen2.5-7b-A", "qwen2.5-7b-B" ], "separated_uncorrected_only_pairs": [ [ "llama3.2-3b-A", "mistral-7b-A" ], [ "mistral-7b-A", "qwen2.5-7b-A" ] ], "primary_arm": "A", "primary_arm_subject_count": 4, "build_count": 4, "arm_count": 2, "by_arm": { "A": { "subjects": [ "gemma2-9b-A", "llama3.2-3b-A", "mistral-7b-A", "qwen2.5-7b-A" ], "comparison_basis": "corrected", "separated_subjects": [ "gemma2-9b-A" ], "not_separated_groups": [ [ "llama3.2-3b-A", "mistral-7b-A", "qwen2.5-7b-A" ] ], "points": "separated subjects only" }, "B": { "subjects": [ "gemma2-9b-B", "llama3.2-3b-B", "mistral-7b-B", "qwen2.5-7b-B" ], "comparison_basis": "none", "separated_subjects": [], "not_separated_groups": [], "points": "none" } }, "display_rule": { "amendment": 16, "text": "The pre-registration reports an uncorrected 95% interval and a Bonferroni-corrected interval for each primary comparison and promotes neither. Showing a point is a more consequential public act than reporting a range, so the point display takes the stricter evidence: in the primary arm only a subject separated from every other subject after correction may show a point; subjects not separated after correction, and every variant of the secondary one-trial arm, show ranges only; no point difference that involves a withheld subject is shown. A pair separated only without correction may be stated in words with its uncorrected range, never as a point or an ordering." }, "point_withheld": [ "not_separated_group_range", "not_separated_group_ranges" ] } }