{ "schema": "compassion-benchmark/model-benchmark-index/1", "program": { "name": "AI Model Compassion Benchmark", "status": "No model has an official score. Everything published so far is an unofficial pilot: not a score, not a ranking, comparability none.", "official_scores": false, "comparability": "none", "independence": "No model developer pays for inclusion, a result, or the withholding of a finding.", "page_url": "https://compassionbenchmark.com/ai-models" }, "methodology": { "url": "https://compassionbenchmark.com/ai-models/methodology", "bank_version": "v2.0", "items_total": 93, "items_served_by_default": 83, "crisis_items_excluded_by_default": 5, "dimensions": 8, "subdimensions": 40, "trials_in_default_run": 249, "minimum_trials_per_item": 3, "note": "Every item is published with its full answer key, so a model trained since publication may have memorised it. A result on this bank is not comparable across models." }, "cb_probe": { "name": "cb-probe", "summary": "A local MCP server (stdio) that lets a model run the published items on itself. No network call and no API key. A self-run is unofficial, an upper bound, and never comparable.", "version": "0.3.0", "license": "MIT", "repository": "https://github.com/Klingdom/applied-compassion-benchmark", "tool_count": 12, "tools": [ "list_probe_items", "get_anchors", "open_judge_session", "record_item_estimate", "summarise_judge_session", "explain_what_this_is_not", "start_scored_run", "next_item", "run_status", "record_item_rating", "run_exposure_probe", "finish_scored_run" ], "install_doc_url": "https://compassionbenchmark.com/ai-evaluation-suite#mcp-server" }, "pilot_reports": [ { "run_id": "pilot-2026-10-03", "title": "A third look: four open-weight builds, each run two ways", "date": "2026-10-03", "status": "unofficial pilot", "official": false, "comparability": "none", "decision_ref": "D-29a", "access_tier": "local-open-weight (Ollama, 4-bit quantised builds pinned by digest)", "report_url": "https://compassionbenchmark.com/ai-models/reports/pilot-2026-10-03", "markdown_url": "https://compassionbenchmark.com/ai-models/reports/pilot-2026-10-03.md", "wave_url": "https://compassionbenchmark.com/data/model-waves/pilot-2026-10-03.json", "subjects": [ { "label": "gemma2-9b-A", "arm": "A", "trials": 3, "provenance": { "developer": "Google", "licence": "Gemma Terms of Use", "parameter_size": "9.2B", "quantisation": "Q4_0", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "google", "build_tag": "gemma2:9b", "digest_sha256": "ff02c3702f322b9e075e9568332d96c0a7028002f1a5a056e0a6784320a4db0b" } }, { "label": "gemma2-9b-B", "arm": "B", "trials": 1, "provenance": { "developer": "Google", "licence": "Gemma Terms of Use", "parameter_size": "9.2B", "quantisation": "Q4_0", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "google", "build_tag": "gemma2:9b", "digest_sha256": "ff02c3702f322b9e075e9568332d96c0a7028002f1a5a056e0a6784320a4db0b" } }, { "label": "llama3.2-3b-A", "arm": "A", "trials": 3, "provenance": { "developer": "Meta", "licence": "Llama 3.2 Community Licence", "parameter_size": "3.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "meta", "build_tag": "llama3.2:latest", "digest_sha256": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72" } }, { "label": "llama3.2-3b-B", "arm": "B", "trials": 1, "provenance": { "developer": "Meta", "licence": "Llama 3.2 Community Licence", "parameter_size": "3.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "meta", "build_tag": "llama3.2:latest", "digest_sha256": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72" } }, { "label": "mistral-7b-A", "arm": "A", "trials": 3, "provenance": { "developer": "Mistral AI", "licence": "Apache-2.0", "parameter_size": "7.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "mistral", "build_tag": "mistral:7b", "digest_sha256": "6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf" } }, { "label": "mistral-7b-B", "arm": "B", "trials": 1, "provenance": { "developer": "Mistral AI", "licence": "Apache-2.0", "parameter_size": "7.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "mistral", "build_tag": "mistral:7b", "digest_sha256": "6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf" } }, { "label": "qwen2.5-7b-A", "arm": "A", "trials": 3, "provenance": { "developer": "Alibaba (Qwen)", "licence": "Apache-2.0", "parameter_size": "7.6B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "qwen", "build_tag": "qwen2.5:7b", "digest_sha256": "845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e" } }, { "label": "qwen2.5-7b-B", "arm": "B", "trials": 1, "provenance": { "developer": "Alibaba (Qwen)", "licence": "Apache-2.0", "parameter_size": "7.6B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "qwen", "build_tag": "qwen2.5:7b", "digest_sha256": "845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e" } } ], "separation": { "summary": "In the primary arm (no length instruction), after correction for multiple comparisons, the test separated gemma2-9b-A from the other three variants and could not tell llama3.2-3b-A, mistral-7b-A and qwen2.5-7b-A apart (alphabetical order). llama3.2-3b-A and mistral-7b-A and mistral-7b-A and qwen2.5-7b-A were each separated only without correction for multiple comparisons; that is not stated as a separation. The secondary arm (length instruction, one trial per item) shows ranges only: gemma2-9b-B, llama3.2-3b-B, mistral-7b-B and qwen2.5-7b-B. Not separated is not the same as equal; the pilot cannot say which is higher.", "not_separated_groups": [ [ "llama3.2-3b-A", "mistral-7b-A", "qwen2.5-7b-A" ] ], "separated_subjects": [ "gemma2-9b-A" ], "point_estimates_withheld_for": [ "gemma2-9b-B", "llama3.2-3b-A", "llama3.2-3b-B", "mistral-7b-A", "mistral-7b-B", "qwen2.5-7b-A", "qwen2.5-7b-B" ], "note": "Ranges for every subject are in wave_url. Point estimates are withheld for every variant except one the corrected comparison separated (D-29a, template amendments 2 and 16).", "display_rule": "The pre-registration reports an uncorrected 95% interval and a Bonferroni-corrected interval for each primary comparison and promotes neither. Showing a point is a more consequential public act than reporting a range, so the point display takes the stricter evidence: in the primary arm only a subject separated from every other subject after correction may show a point; subjects not separated after correction, and every variant of the secondary one-trial arm, show ranges only; no point difference that involves a withheld subject is shown. A pair separated only without correction may be stated in words with its uncorrected range, never as a point or an ordering.", "range_only_subjects": [ "gemma2-9b-B", "llama3.2-3b-B", "mistral-7b-B", "qwen2.5-7b-B" ] }, "caveats": [ "Unofficial pilot. It is not a Compassion Benchmark score, its comparability is none, and it supports no comparison with models outside it. No AI model has an official score.", "Models the test could not separate are shown as ranges only. Their point estimates are withheld because their order would read as a ranking. Not separated is not the same as equal.", "The judges were models from a different family than the models tested, not people. The item bank was authored with help from the judges' family.", "Access tier: local-open-weight (Ollama, 4-bit quantised builds pinned by digest).", "0 of 93 bank items have passed human review; 10 were not served in this pilot.", "Crisis-adjacent items were not served, so the pilot says nothing about how any model responds to a person in crisis.", "Reply length is an unresolved confound for any separated model.", "Absolute levels depend on which judges are used; the pilot did not test whether the separation pattern holds under a different choice of judges. Its sensitivity check, which varied no judge, left the separation pattern unchanged.", "Each build was run twice, with and without a length instruction. Only the arm without it carries the corrected comparison; the other arm has one trial per item and shows ranges only.", "The system message is explicit and identical within an arm, but its place in the prompt differs by each build's own template. The secondary arm has no server scorecard because the cb-probe server needs more trials per item; its composites are computed from the same ratings.", "2 deviation(s) from the plan are recorded in the report.", "The item bank is published with its answer key, so a model may have seen the items. Contamination probes ran on sampled items only." ], "preregistration_sha256": "47994b94612d124c5f463beb3c88f295a2cf2c8cc11889899db71cf9ae350f22" }, { "run_id": "pilot-2026-10-02", "title": "A second look: two open-weight models, judged from outside their family", "date": "2026-10-02", "status": "unofficial pilot", "official": false, "comparability": "none", "decision_ref": "D-29a", "access_tier": "local-open-weight (Ollama, 4-bit quantised builds pinned by digest)", "report_url": "https://compassionbenchmark.com/ai-models/reports/pilot-2026-10-02", "markdown_url": "https://compassionbenchmark.com/ai-models/reports/pilot-2026-10-02.md", "wave_url": "https://compassionbenchmark.com/data/model-waves/pilot-2026-10-02.json", "subjects": [ { "label": "llama3.2-3b", "provenance": { "developer": "Meta", "licence": "Llama 3.2 Community Licence", "parameter_size": "3.2B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "llama", "build_tag": "llama3.2:latest", "digest_sha256": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72" } }, { "label": "qwen2.5-7b", "provenance": { "developer": "Alibaba (Qwen)", "licence": "Apache-2.0", "parameter_size": "7.6B", "quantisation": "Q4_K_M", "runtime": "Ollama", "runtime_version": "0.35.0", "model_family": "qwen", "build_tag": "qwen2.5:7b", "digest_sha256": "845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e" } } ], "separation": { "summary": "The test could not tell the two models apart: llama3.2-3b and qwen2.5-7b (alphabetical order). Not separated is not the same as equal; the pilot cannot say which is higher.", "not_separated_groups": [ [ "llama3.2-3b", "qwen2.5-7b" ] ], "separated_subjects": [], "point_estimates_withheld_for": [ "llama3.2-3b", "qwen2.5-7b" ], "note": "Ranges for every subject are in wave_url. Point estimates for not-separated models are withheld (D-29a, template amendment 2)." }, "caveats": [ "Unofficial pilot. It is not a Compassion Benchmark score, its comparability is none, and it supports no comparison with models outside it. No AI model has an official score.", "Models the test could not separate are shown as ranges only. Their point estimates are withheld because their order would read as a ranking. Not separated is not the same as equal.", "The judges were models from a different family than the models tested, not people. The item bank was authored with help from the judges' family.", "Access tier: local-open-weight (Ollama, 4-bit quantised builds pinned by digest).", "0 of 93 bank items have passed human review; 10 were not served in this pilot.", "Crisis-adjacent items were not served, so the pilot says nothing about how any model responds to a person in crisis.", "Reply length is a confound for any rated response and is unresolved.", "Absolute levels depend on which judges are used; the pilot did not test whether the separation pattern holds under a different choice of judges. Its sensitivity check, which varied no judge, left the separation pattern unchanged.", "The pre-registration file was not committed or independently time-stamped before the data existed.", "3 deviation(s) from the plan are recorded in the report.", "The item bank is published with its answer key, so a model may have seen the items. Contamination probes ran on sampled items only." ], "preregistration_sha256": "bb1f11b9d23c72e0be2a928f6dc3a5a712e46a19c38c34dc59813596e6de1415" }, { "run_id": "pilot-2026-10-01", "title": "We tested the test: a first look at measuring compassion in AI models", "date": "2026-10-01", "status": "unofficial pilot", "official": false, "comparability": "none", "decision_ref": "D-29a", "access_tier": "agent (Claude Code subagent; snapshot id unverifiable)", "report_url": "https://compassionbenchmark.com/ai-models/reports/pilot-2026-10-01", "markdown_url": "https://compassionbenchmark.com/ai-models/reports/pilot-2026-10-01.md", "wave_url": "https://compassionbenchmark.com/data/model-waves/pilot-2026-10-01.json", "subjects": [ { "label": "claude-fable", "provenance": { "access_tier": "agent (Claude Code subagent; snapshot id unverifiable)" } }, { "label": "claude-haiku", "provenance": { "access_tier": "agent (Claude Code subagent; snapshot id unverifiable)" } }, { "label": "claude-opus", "provenance": { "access_tier": "agent (Claude Code subagent; snapshot id unverifiable)" } }, { "label": "claude-sonnet", "provenance": { "access_tier": "agent (Claude Code subagent; snapshot id unverifiable)" } } ], "separation": { "summary": "The test separated claude-haiku from the other three models. It could not tell claude-fable, claude-opus and claude-sonnet apart. Not separated is not the same as equal; the pilot cannot say which is higher.", "not_separated_groups": [ [ "claude-fable", "claude-opus", "claude-sonnet" ] ], "separated_subjects": [ "claude-haiku" ], "point_estimates_withheld_for": [ "claude-fable", "claude-opus", "claude-sonnet" ], "note": "Ranges for every subject are in wave_url. Point estimates for not-separated models are withheld (D-29a, template amendment 2)." }, "caveats": [ "Unofficial pilot. It is not a Compassion Benchmark score, its comparability is none, and it supports no comparison with models outside it. No AI model has an official score.", "Models the test could not separate are shown as ranges only. Their point estimates are withheld because their order would read as a ranking. Not separated is not the same as equal.", "The judges were models, not people, and came from the same family as the models tested, so they may share preferences the pilot cannot measure.", "Access tier: agent (Claude Code subagent; snapshot id unverifiable).", "0 of 93 bank items have passed human review; 10 were not served in this pilot.", "Crisis-adjacent items were not served, so the pilot says nothing about how any model responds to a person in crisis.", "A model the test separated also wrote much shorter replies; reply length is an unresolved confound, and the pilot cannot say whether the gap is about compassion or about length.", "One judge was excluded after the fact (a disclosed post-hoc protocol change); the report describes it.", "Absolute levels depend on which judges are used; the separation pattern was checked against the judge choice and did not change.", "The item bank is published with its answer key, so a model may have seen the items. Contamination probes ran on sampled items only." ] } ], "links": { "reports_index": "https://compassionbenchmark.com/ai-models/reports", "llms_txt": "https://compassionbenchmark.com/llms.txt", "llms_full_txt": "https://compassionbenchmark.com/llms-full.txt", "well_known": "https://compassionbenchmark.com/.well-known/compassion-benchmark.json", "sitemap": "https://compassionbenchmark.com/sitemap.xml" } }